<!DOCTYPE html><html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" data-whc_version="25.0">
    <head><link rel="shortcut icon" href="../../../oxygen-webhelp/template/images/favicon.png"/><link rel="icon" href="../../../oxygen-webhelp/template/images/favicon.png"/><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="copyright" content="(C) Copyright 2024"/><meta name="generator" content="DITA-OT"/><meta name="description" content="Supported pipeline types: Data Collector The Azure Data Lake Storage Gen1 destination writes data to Microsoft Azure Data Lake Storage Gen1. You can use the Azure Data Lake Storage Gen1 destination in ..."/><meta name="prodname" content="Data Collector"/><meta name="version" content="3"/><meta name="release" content="16"/><meta name="modification" content="0"/>        
      <title>Azure Data Lake Storage Gen1 (deprecated)</title><!--  Generated with Oxygen version 25.1, build number 2023042410.  --><meta name="wh-path2root" content="../../../"/><meta name="wh-toc-id" content="concept_xzc_wfq_xhb-d16893e44968"/><meta name="wh-source-relpath" content="datacollector/UserGuide/Destinations/ADLS-G1-D.dita"/><meta name="wh-out-relpath" content="datacollector/UserGuide/Destinations/ADLS-G1-D.html"/>

    <link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/app/commons.css?buildId=2023042410"/>
    <link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/app/topic.css?buildId=2023042410"/>

    <script src="../../../oxygen-webhelp/app/options/properties.js?buildId=20240802104629"></script>
    <script src="../../../oxygen-webhelp/app/localization/strings.js?buildId=2023042410"></script>
    <script src="../../../oxygen-webhelp/app/search/index/keywords.js?buildId=20240802104629"></script>
    <script defer="defer" src="../../../oxygen-webhelp/app/commons.js?buildId=2023042410"></script>
    <script defer="defer" src="../../../oxygen-webhelp/app/topic.js?buildId=2023042410"></script>
<link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/template/light.css?buildId=2023042410"/><link rel="stylesheet" type="text/css" href="../../../skin.css"/></head>

    <body class="wh_topic_page frmBody">
        
        
        

        
<nav class="navbar navbar-default wh_header" data-whc_version="25.0">
    <div class="container-fluid">
        <div class="wh_header_flex_container navbar-nav navbar-expand-md navbar-dark">
            <div class="wh_logo_and_publication_title_container">
                <div class="wh_logo_and_publication_title">
                    
                    <!--
                            This component will be generated when the next parameters are specified in the transformation scenario:
                            'webhelp.logo.image' and 'webhelp.logo.image.target.url'.
                            See: http://oxygenxml.com/doc/versions/17.1/ug-editor/#topics/dita_webhelp_output.html.
                    -->
                    
                    <div class=" wh_publication_title "><a href="../../../index.html"><span class="booktitle">  <span class="ph mainbooktitle"><span class="ph">Data Collector</span> User Guide</span>  </span></a></div>
                    
                </div>
                
                <!-- The menu button for mobile devices is copied in the output only when the 'webhelp.show.top.menu' parameter is set to 'yes' -->
                
            </div>

            <div class="wh_top_menu_and_indexterms_link collapse navbar-collapse">
                
                
                <div class=" wh_indexterms_link "><a href="../../../indexTerms.html" title="Index" aria-label="Go to index terms page"><span>Index</span></a></div>
                
            </div>
        </div>
    </div>
</nav>

        <div class=" wh_search_input navbar-form wh_topic_page_search search " role="form">


<form id="searchForm" method="get" role="search" action="../../../search.html"><div><input type="search" placeholder="Search " class="wh_search_textfield" id="textToSearch" name="searchQuery" aria-label="Search query" required="required"/><button type="submit" class="wh_search_button" aria-label="Search"><span class="search_input_text">Search</span></button></div></form>

</div>
        
        <div class="container-fluid">
            <div class="row">

                <nav class="wh_tools d-print-none">
                    
<div data-tooltip-position="bottom" class=" wh_breadcrumb "><ol class="d-print-none"><li><span class="home"><a href="../../../index.html"><span>Home</span></a></span></li><li><div class="topicref" data-id="concept_agj_cfj_br"><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Destinations-title.html">Destinations</a></div></div></li><li class="active"><div class="topicref" data-id="concept_xzc_wfq_xhb"><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_xzc_wfq_xhb">Azure Data Lake Storage Gen1 (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li></ol></div>



                    <div class="wh_right_tools "><button class="wh_hide_highlight" aria-label="Toggle search highlights" title="Toggle search highlights"></button><button class="webhelp_expand_collapse_sections" data-next-state="collapsed" aria-label="Collapse sections" title="Collapse sections"></button><div class=" wh_navigation_links "><span id="topic_navigation_links" class="navheader">
  
<span class="navprev"><a class="- topic/link link" href="../../../datacollector/UserGuide/Destinations/DataLakeStore.html#concept_jzm_kf4_zx" title="Azure Data Lake Storage (Legacy) (deprecated)" aria-label="Previous topic: Azure Data Lake Storage (Legacy) (deprecated)" rel="prev"></a></span>  
<span class="navnext"><a class="- topic/link link" href="../../../datacollector/UserGuide/Destinations/ADLS-G2-D.html#concept_ajp_1d2_vhb" title="Azure Data Lake Storage Gen2" aria-label="Next topic: Azure Data Lake Storage Gen2" rel="next"></a></span>  </span></div>
<!--External resource link-->
<div class=" wh_print_link print d-none d-md-inline-block "><button onClick="window.print()" title="Print this page" aria-label="Print this page"></button></div>
                        
                        
                        
                        
                    </div>
                </nav>
            </div>

            

<div class="wh_content_area">
                <div class="row">
                    


                        <nav role="navigation" id="wh_publication_toc" class="col-lg-3 col-md-3 col-sm-12 d-md-block d-none d-print-none">
<div id="wh_publication_toc_content">


                            <div class=" wh_publication_toc " data-tooltip-position="right"><span class="expand-button-action-labels"><span id="button-expand-action" role="button" aria-label="Expand"></span><span id="button-collapse-action" role="button" aria-label="Collapse"></span><span id="button-pending-action" role="button" aria-label="Pending"></span></span><ul role="tree" aria-label="Table of Contents"><li role="treeitem" aria-expanded="false"><div data-tocid="concept_htw_ghg_jq-d16893e53" class="topicref" data-id="concept_htw_ghg_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_htw_ghg_jq-d16893e53-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Getting_Started/GettingStarted_Title.html#concept_htw_ghg_jq" id="concept_htw_ghg_jq-d16893e53-link">Getting Started</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_l2v_nlp_mpb-d16893e331" class="topicref" data-id="concept_l2v_nlp_mpb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_l2v_nlp_mpb-d16893e331-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/ReleaseNotes/ReleaseNotes.html#concept_l2v_nlp_mpb" id="concept_l2v_nlp_mpb-d16893e331-link">Release Notes</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_l4q_flb_kr-d16893e2582" class="topicref" data-id="concept_l4q_flb_kr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_l4q_flb_kr-d16893e2582-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Installation/Install_title.html" id="concept_l4q_flb_kr-d16893e2582-link">Installation</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ylh_yyz_ky-d16893e3984" class="topicref" data-id="concept_ylh_yyz_ky" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ylh_yyz_ky-d16893e3984-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Configuration/Config_title.html" id="concept_ylh_yyz_ky-d16893e3984-link">Configuration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ejk_f1f_5v-d16893e7058" class="topicref" data-id="concept_ejk_f1f_5v" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ejk_f1f_5v-d16893e7058-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Upgrade/Upgrade_title.html" id="concept_ejk_f1f_5v-d16893e7058-link">Upgrade</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qsw_cjy_bt-d16893e10103" class="topicref" data-id="concept_qsw_cjy_bt" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qsw_cjy_bt-d16893e10103-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Design/PipelineDesign_title.html" id="concept_qsw_cjy_bt-d16893e10103-link">Pipeline Concepts and Design</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qn1_wn4_kq-d16893e11199" class="topicref" data-id="concept_qn1_wn4_kq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qn1_wn4_kq-d16893e11199-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Configuration/PipelineConfiguration_title.html" id="concept_qn1_wn4_kq-d16893e11199-link">Pipeline Configuration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hdr_gyw_41b-d16893e13057" class="topicref" data-id="concept_hdr_gyw_41b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hdr_gyw_41b-d16893e13057-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Data_Formats/DataFormats-Title.html" id="concept_hdr_gyw_41b-d16893e13057-link">Data Formats</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yjl_nc5_jq-d16893e14164" class="topicref" data-id="concept_yjl_nc5_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yjl_nc5_jq-d16893e14164-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Origins/Origins_title.html" id="concept_yjl_nc5_jq-d16893e14164-link">Origins</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yjl_nc5_jq-d16893e35197" class="topicref" data-id="concept_yjl_nc5_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yjl_nc5_jq-d16893e35197-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Processors_title.html" id="concept_yjl_nc5_jq-d16893e35197-link">Processors</a></div></div></li><li role="treeitem" aria-expanded="true"><div data-tocid="concept_agj_cfj_br-d16893e44037" class="topicref" data-id="concept_agj_cfj_br" data-state="expanded"><span role="button" tabindex="0" aria-labelledby="button-collapse-action concept_agj_cfj_br-d16893e44037-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Destinations-title.html" id="concept_agj_cfj_br-d16893e44037-link">Destinations</a></div></div><ul role="group" class="navbar-nav nav-list"><li role="treeitem"><div data-tocid="concept_hpr_twm_jq-d16893e44059" class="topicref" data-id="concept_hpr_twm_jq" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Destinations_overview.html#concept_hpr_twm_jq" id="concept_hpr_twm_jq-d16893e44059-link">Destinations</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_gyq_rpr_4cb-d16893e44083" class="topicref" data-id="concept_gyq_rpr_4cb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_gyq_rpr_4cb-d16893e44083-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Aerospike.html#concept_gyq_rpr_4cb" id="concept_gyq_rpr_4cb-d16893e44083-link">Aerospike (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ibv_yj4_qxb-d16893e44129" class="topicref" data-id="concept_ibv_yj4_qxb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ibv_yj4_qxb-d16893e44129-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/AerospikeClient.html#concept_ibv_yj4_qxb" id="concept_ibv_yj4_qxb-d16893e44129-link">Aerospike Client</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_avx_bnq_rt-d16893e44245" class="topicref" data-id="concept_avx_bnq_rt" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_avx_bnq_rt-d16893e44245-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/AmazonS3.html#concept_avx_bnq_rt" id="concept_avx_bnq_rt-d16893e44245-link">Amazon S3</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ggg_4wy_41c-d16893e44533" class="topicref" data-id="concept_ggg_4wy_41c" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ggg_4wy_41c-d16893e44533-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/AzureBlobStorage.html#concept_ggg_4wy_41c" id="concept_ggg_4wy_41c-d16893e44533-link">Azure Blob Storage</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jzm_kf4_zx-d16893e44699" class="topicref" data-id="concept_jzm_kf4_zx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jzm_kf4_zx-d16893e44699-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/DataLakeStore.html#concept_jzm_kf4_zx" id="concept_jzm_kf4_zx-d16893e44699-link">Azure Data Lake Storage (Legacy) (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="true" class="active"><div data-tocid="concept_xzc_wfq_xhb-d16893e44968" class="topicref" data-id="concept_xzc_wfq_xhb" data-state="expanded"><span role="button" tabindex="0" aria-labelledby="button-collapse-action concept_xzc_wfq_xhb-d16893e44968-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_xzc_wfq_xhb" id="concept_xzc_wfq_xhb-d16893e44968-link">Azure Data Lake Storage Gen1 (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div><ul role="group" class="navbar-nav nav-list"><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fn4_qyw_zhb-d16893e44992" class="topicref" data-id="concept_fn4_qyw_zhb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fn4_qyw_zhb-d16893e44992-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_fn4_qyw_zhb" id="concept_fn4_qyw_zhb-d16893e44992-link">Prerequisites</a></div></div></li><li role="treeitem"><div data-tocid="concept_e5y_2zw_zhb-d16893e45036" class="topicref" data-id="concept_e5y_2zw_zhb" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_e5y_2zw_zhb" id="concept_e5y_2zw_zhb-d16893e45036-link">Directory Templates</a></div></div></li><li role="treeitem"><div data-tocid="concept_wps_x2x_zhb-d16893e45058" class="topicref" data-id="concept_wps_x2x_zhb" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_wps_x2x_zhb" id="concept_wps_x2x_zhb-d16893e45058-link">Time Basis</a></div></div></li><li role="treeitem"><div data-tocid="concept_x35_tgx_zhb-d16893e45080" class="topicref" data-id="concept_x35_tgx_zhb" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_x35_tgx_zhb" id="concept_x35_tgx_zhb-d16893e45080-link">Late Records and Late Record Handling</a></div></div></li><li role="treeitem"><div data-tocid="concept_kyy_chx_zhb-d16893e45102" class="topicref" data-id="concept_kyy_chx_zhb" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_kyy_chx_zhb" id="concept_kyy_chx_zhb-d16893e45102-link">Timeout to Close Idle Files</a></div></div></li><li role="treeitem"><div data-tocid="concept_uyx_3bt_13b-d16893e45124" class="topicref" data-id="concept_uyx_3bt_13b" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_uyx_3bt_13b" id="concept_uyx_3bt_13b-d16893e45124-link">Recovery</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_et1_lhx_zhb-d16893e45146" class="topicref" data-id="concept_et1_lhx_zhb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_et1_lhx_zhb-d16893e45146-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_et1_lhx_zhb" id="concept_et1_lhx_zhb-d16893e45146-link">Event Generation</a></div></div></li><li role="treeitem"><div data-tocid="concept_ymd_3fz_j4b-d16893e45190" class="topicref" data-id="concept_ymd_3fz_j4b" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_ymd_3fz_j4b" id="concept_ymd_3fz_j4b-d16893e45190-link">Resolving Out of Memory Errors</a></div></div></li><li role="treeitem"><div data-tocid="concept_wld_zhx_zhb-d16893e45212" class="topicref" data-id="concept_wld_zhx_zhb" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#concept_wld_zhx_zhb" id="concept_wld_zhx_zhb-d16893e45212-link">Data Formats</a></div></div></li><li role="treeitem"><div data-tocid="task_nx5_4t5_5hb-d16893e45234" class="topicref" data-id="task_nx5_4t5_5hb" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G1-D.html#task_nx5_4t5_5hb" id="task_nx5_4t5_5hb-d16893e45234-link">Configuring an Azure Data Lake Storage Gen1 Destination</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li></ul></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ajp_1d2_vhb-d16893e45258" class="topicref" data-id="concept_ajp_1d2_vhb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ajp_1d2_vhb-d16893e45258-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ADLS-G2-D.html#concept_ajp_1d2_vhb" id="concept_ajp_1d2_vhb-d16893e45258-link">Azure Data Lake Storage Gen2</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xq5_d5q_1bb-d16893e45549" class="topicref" data-id="concept_xq5_d5q_1bb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xq5_d5q_1bb-d16893e45549-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/AzureEventHubProducer.html#concept_xq5_d5q_1bb" id="concept_xq5_d5q_1bb-d16893e45549-link">Azure Event Hub Producer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pnd_jkq_1bb-d16893e45617" class="topicref" data-id="concept_pnd_jkq_1bb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pnd_jkq_1bb-d16893e45617-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/AzureIoTHub.html#concept_pnd_jkq_1bb" id="concept_pnd_jkq_1bb-d16893e45617-link">Azure IoT Hub Producer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hjv_5nn_r3b-d16893e45716" class="topicref" data-id="concept_hjv_5nn_r3b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hjv_5nn_r3b-d16893e45716-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/AzureSynapse.html#concept_hjv_5nn_r3b" id="concept_hjv_5nn_r3b-d16893e45716-link">Azure Synapse SQL</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hfy_mfd_sr-d16893e46145" class="topicref" data-id="concept_hfy_mfd_sr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hfy_mfd_sr-d16893e46145-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Cassandra.html#concept_hfy_mfd_sr" id="concept_hfy_mfd_sr-d16893e46145-link">Cassandra</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hw5_s3n_sz-d16893e46279" class="topicref" data-id="concept_hw5_s3n_sz" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hw5_s3n_sz-d16893e46279-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/CoAPClient.html#concept_hw5_s3n_sz" id="concept_hw5_s3n_sz-d16893e46279-link">CoAP Client</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ahq_1wq_h2b-d16893e46347" class="topicref" data-id="concept_ahq_1wq_h2b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ahq_1wq_h2b-d16893e46347-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Couchbase.html#concept_ahq_1wq_h2b" id="concept_ahq_1wq_h2b-d16893e46347-link">Couchbase</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ddy_cdz_clb-d16893e46461" class="topicref" data-id="concept_ddy_cdz_clb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ddy_cdz_clb-d16893e46461-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/DeltaLake.html#concept_ddy_cdz_clb" id="concept_ddy_cdz_clb-d16893e46461-link">Databricks Delta Lake</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_u5t_vpv_4r-d16893e46869" class="topicref" data-id="concept_u5t_vpv_4r" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_u5t_vpv_4r-d16893e46869-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Elasticsearch.html#concept_u5t_vpv_4r" id="concept_u5t_vpv_4r-d16893e46869-link">Elasticsearch</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pzn_hl4_yr-d16893e47009" class="topicref" data-id="concept_pzn_hl4_yr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pzn_hl4_yr-d16893e47009-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Flume.html#concept_pzn_hl4_yr" id="concept_pzn_hl4_yr-d16893e47009-link">Flume (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hj4_brk_dbb-d16893e47077" class="topicref" data-id="concept_hj4_brk_dbb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hj4_brk_dbb-d16893e47077-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/BigQuery.html#concept_hj4_brk_dbb" id="concept_hj4_brk_dbb-d16893e47077-link">Google BigQuery (Legacy) (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_nfr_by1_3rb-d16893e47173" class="topicref" data-id="concept_nfr_by1_3rb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_nfr_by1_3rb-d16893e47173-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/GBigQuery.html#concept_nfr_by1_3rb" id="concept_nfr_by1_3rb-d16893e47173-link">Google BigQuery</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pl5_tmq_tx-d16893e47559" class="topicref" data-id="concept_pl5_tmq_tx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pl5_tmq_tx-d16893e47559-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Bigtable.html#concept_pl5_tmq_tx" id="concept_pl5_tmq_tx-d16893e47559-link">Google Bigtable</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_p4n_jrl_nbb-d16893e47777" class="topicref" data-id="concept_p4n_jrl_nbb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_p4n_jrl_nbb-d16893e47777-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/GCS.html#concept_p4n_jrl_nbb" id="concept_p4n_jrl_nbb-d16893e47777-link">Google Cloud Storage</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qsj_hk1_v1b-d16893e48000" class="topicref" data-id="concept_qsj_hk1_v1b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qsj_hk1_v1b-d16893e48000-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/PubSubPublisher.html#concept_qsj_hk1_v1b" id="concept_qsj_hk1_v1b-d16893e48000-link">Google Pub/Sub Publisher</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qjf_xdz_q3b-d16893e48090" class="topicref" data-id="concept_qjf_xdz_q3b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qjf_xdz_q3b-d16893e48090-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/GPSS.html#concept_qjf_xdz_q3b" id="concept_qjf_xdz_q3b-d16893e48090-link">GPSS Producer (deprecated)</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_awl_4km_zq-d16893e48226" class="topicref" data-id="concept_awl_4km_zq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_awl_4km_zq-d16893e48226-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/HadoopFS-destination.html#concept_awl_4km_zq" id="concept_awl_4km_zq-d16893e48226-link">Hadoop FS</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wsz_5t5_vr-d16893e48561" class="topicref" data-id="concept_wsz_5t5_vr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wsz_5t5_vr-d16893e48561-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/HBase.html#concept_wsz_5t5_vr" id="concept_wsz_5t5_vr-d16893e48561-link">HBase</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_gcr_z2t_zv-d16893e48732" class="topicref" data-id="concept_gcr_z2t_zv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_gcr_z2t_zv-d16893e48732-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/HiveMetastore.html#concept_gcr_z2t_zv" id="concept_gcr_z2t_zv-d16893e48732-link">Hive Metastore</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_kvs_3hh_ht-d16893e48985" class="topicref" data-id="concept_kvs_3hh_ht" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_kvs_3hh_ht-d16893e48985-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Hive.html#concept_kvs_3hh_ht" id="concept_kvs_3hh_ht-d16893e48985-link">Hive Streaming (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_khl_sg5_lz-d16893e49053" class="topicref" data-id="concept_khl_sg5_lz" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_khl_sg5_lz-d16893e49053-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/HTTPClient.html#concept_khl_sg5_lz" id="concept_khl_sg5_lz-d16893e49053-link">HTTP Client</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_inf_db_sr-d16893e49363" class="topicref" data-id="concept_inf_db_sr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_inf_db_sr-d16893e49363-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/InfluxDB.html#concept_inf_db_sr" id="concept_inf_db_sr-d16893e49363-link">InfluxDB</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_s4q_mfn_crb-d16893e49409" class="topicref" data-id="concept_s4q_mfn_crb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_s4q_mfn_crb-d16893e49409-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/InfluxDB2.html#concept_s4q_mfn_crb" id="concept_s4q_mfn_crb-d16893e49409-link">InfluxDB 2.x</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_kvs_3hh_ht-d16893e49457" class="topicref" data-id="concept_kvs_3hh_ht" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_kvs_3hh_ht-d16893e49457-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/JDBCProducer.html#concept_kvs_3hh_ht" id="concept_kvs_3hh_ht-d16893e49457-link">JDBC Producer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yrx_lr3_qbc-d16893e49621" class="topicref" data-id="concept_yrx_lr3_qbc" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yrx_lr3_qbc-d16893e49621-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/JiraD.html#concept_yrx_lr3_qbc" id="concept_yrx_lr3_qbc-d16893e49621-link">Jira</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_sfz_ww5_n1b-d16893e49742" class="topicref" data-id="concept_sfz_ww5_n1b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_sfz_ww5_n1b-d16893e49742-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/JMSProducer.html#concept_sfz_ww5_n1b" id="concept_sfz_ww5_n1b-d16893e49742-link">JMS Producer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_oq2_5jl_zq-d16893e49923" class="topicref" data-id="concept_oq2_5jl_zq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_oq2_5jl_zq-d16893e49923-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/KProducer.html#concept_oq2_5jl_zq" id="concept_oq2_5jl_zq-d16893e49923-link">Kafka Producer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_bjv_dpk_kv-d16893e50153" class="topicref" data-id="concept_bjv_dpk_kv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_bjv_dpk_kv-d16893e50153-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/KinFirehose.html#concept_bjv_dpk_kv" id="concept_bjv_dpk_kv-d16893e50153-link">Kinesis Firehose</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_swk_h1j_yr-d16893e50268" class="topicref" data-id="concept_swk_h1j_yr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_swk_h1j_yr-d16893e50268-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/KinProducer.html#concept_swk_h1j_yr" id="concept_swk_h1j_yr-d16893e50268-link">Kinesis Producer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hxh_5xg_qbb-d16893e50402" class="topicref" data-id="concept_hxh_5xg_qbb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hxh_5xg_qbb-d16893e50402-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/KineticaDB.html#concept_hxh_5xg_qbb" id="concept_hxh_5xg_qbb-d16893e50402-link">KineticaDB (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_chy_xxg_4v-d16893e50492" class="topicref" data-id="concept_chy_xxg_4v" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_chy_xxg_4v-d16893e50492-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Kudu.html#concept_chy_xxg_4v" id="concept_chy_xxg_4v-d16893e50492-link">Kudu </a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_zvc_bv5_1r-d16893e50606" class="topicref" data-id="concept_zvc_bv5_1r" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_zvc_bv5_1r-d16893e50606-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/LocalFS.html#concept_zvc_bv5_1r" id="concept_zvc_bv5_1r-d16893e50606-link">Local FS</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_vxg_w2z_yv-d16893e50828" class="topicref" data-id="concept_vxg_w2z_yv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_vxg_w2z_yv-d16893e50828-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/MapRDB.html#concept_vxg_w2z_yv" id="concept_vxg_w2z_yv-d16893e50828-link">MapR DB</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_i4h_2kj_dy-d16893e50999" class="topicref" data-id="concept_i4h_2kj_dy" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_i4h_2kj_dy-d16893e50999-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/MapRDBJSON.html#concept_i4h_2kj_dy" id="concept_i4h_2kj_dy-d16893e50999-link">MapR DB JSON</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_spv_xlc_fv-d16893e51161" class="topicref" data-id="concept_spv_xlc_fv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_spv_xlc_fv-d16893e51161-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/MapRFS.html#concept_spv_xlc_fv" id="concept_spv_xlc_fv-d16893e51161-link">MapR FS </a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_cfj_qbn_2v-d16893e51459" class="topicref" data-id="concept_cfj_qbn_2v" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_cfj_qbn_2v-d16893e51459-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/MapRStreamsProd.html#concept_cfj_qbn_2v" id="concept_cfj_qbn_2v-d16893e51459-link">MapR Streams Producer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_kvs_3hh_ht-d16893e51606" class="topicref" data-id="concept_kvs_3hh_ht" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_kvs_3hh_ht-d16893e51606-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/MemSQLLoader.html#concept_kvs_3hh_ht" id="concept_kvs_3hh_ht-d16893e51606-link">MemSQL Fast Loader (deprecated)</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_eth_k5n_4v-d16893e51741" class="topicref" data-id="concept_eth_k5n_4v" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_eth_k5n_4v-d16893e51741-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/MongoDB.html#concept_eth_k5n_4v" id="concept_eth_k5n_4v-d16893e51741-link">MongoDB</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_kr3_kqt_n5b-d16893e51875" class="topicref" data-id="concept_kr3_kqt_n5b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_kr3_kqt_n5b-d16893e51875-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/MongoDBAtlas-D.html#concept_kr3_kqt_n5b" id="concept_kr3_kqt_n5b-d16893e51875-link">MongoDB Atlas</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_odz_txt_lz-d16893e52107" class="topicref" data-id="concept_odz_txt_lz" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_odz_txt_lz-d16893e52107-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/MQTTPublisher.html#concept_odz_txt_lz" id="concept_odz_txt_lz-d16893e52107-link">MQTT Publisher</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pl5_tdg_gcb-d16893e52222" class="topicref" data-id="concept_pl5_tdg_gcb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pl5_tdg_gcb-d16893e52222-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/NamedPipe.html#concept_pl5_tdg_gcb" id="concept_pl5_tdg_gcb-d16893e52222-link">Named Pipe</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_shv_2jd_r1c-d16893e52396" class="topicref" data-id="concept_shv_2jd_r1c" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_shv_2jd_r1c-d16893e52396-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Oracle.html#concept_shv_2jd_r1c" id="concept_shv_2jd_r1c-d16893e52396-link">Oracle</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fq3_kpc_r2b-d16893e52564" class="topicref" data-id="concept_fq3_kpc_r2b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fq3_kpc_r2b-d16893e52564-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/PulsarProducer.html#concept_fq3_kpc_r2b" id="concept_fq3_kpc_r2b-d16893e52564-link">Pulsar Producer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pxj_rvy_dv-d16893e52779" class="topicref" data-id="concept_pxj_rvy_dv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pxj_rvy_dv-d16893e52779-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/RabbitMQ.html#concept_pxj_rvy_dv" id="concept_pxj_rvy_dv-d16893e52779-link">RabbitMQ Producer</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ktc_gw2_gw-d16893e52847" class="topicref" data-id="concept_ktc_gw2_gw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ktc_gw2_gw-d16893e52847-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Redis.html#concept_ktc_gw2_gw" id="concept_ktc_gw2_gw-d16893e52847-link">Redis</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_rlb_rt3_rx-d16893e52981" class="topicref" data-id="concept_rlb_rt3_rx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_rlb_rt3_rx-d16893e52981-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Salesforce.html#concept_rlb_rt3_rx" id="concept_rlb_rt3_rx-d16893e52981-link">Salesforce</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_q2c_hz2_zsb-d16893e53120" class="topicref" data-id="concept_q2c_hz2_zsb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_q2c_hz2_zsb-d16893e53120-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/SalesforceBAPI2.html#concept_q2c_hz2_zsb" id="concept_q2c_hz2_zsb-d16893e53120-link">Salesforce Bulk API 2.0 </a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_lfk_hx2_ct-d16893e53261" class="topicref" data-id="concept_lfk_hx2_ct" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_lfk_hx2_ct-d16893e53261-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/SDC_RPCdest.html#concept_lfk_hx2_ct" id="concept_lfk_hx2_ct-d16893e53261-link">SDC RPC (deprecated)</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_eyd_zx4_q2b-d16893e53357" class="topicref" data-id="concept_eyd_zx4_q2b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_eyd_zx4_q2b-d16893e53357-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/SendResponse.html#concept_eyd_zx4_q2b" id="concept_eyd_zx4_q2b-d16893e53357-link">Send Response to Origin</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_sgt_m2m_xhb-d16893e53403" class="topicref" data-id="concept_sgt_m2m_xhb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_sgt_m2m_xhb-d16893e53403-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/SFTP.html#concept_sgt_m2m_xhb" id="concept_sgt_m2m_xhb-d16893e53403-link">SFTP/FTP/FTPS Client</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_rmn_3nj_twb-d16893e53543" class="topicref" data-id="concept_rmn_3nj_twb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_rmn_3nj_twb-d16893e53543-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/SingleStore.html#concept_rmn_3nj_twb" id="concept_rmn_3nj_twb-d16893e53543-link">SingleStore</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_vxl_zzc_1gb-d16893e53609" class="topicref" data-id="concept_vxl_zzc_1gb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_vxl_zzc_1gb-d16893e53609-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Snowflake.html#concept_vxl_zzc_1gb" id="concept_vxl_zzc_1gb-d16893e53609-link">Snowflake</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jmh_p25_4qb-d16893e54127" class="topicref" data-id="concept_jmh_p25_4qb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jmh_p25_4qb-d16893e54127-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/SnowflakeFileUploader.html#concept_jmh_p25_4qb" id="concept_jmh_p25_4qb-d16893e54127-link">Snowflake File Uploader</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_z2g_q1r_wr-d16893e54337" class="topicref" data-id="concept_z2g_q1r_wr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_z2g_q1r_wr-d16893e54337-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Solr.html#concept_z2g_q1r_wr" id="concept_z2g_q1r_wr-d16893e54337-link">Solr</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_zzr_pqn_xdb-d16893e54436" class="topicref" data-id="concept_zzr_pqn_xdb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_zzr_pqn_xdb-d16893e54436-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Splunk.html#concept_zzr_pqn_xdb" id="concept_zzr_pqn_xdb-d16893e54436-link">Splunk</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hjv_5nn_r3b-d16893e54557" class="topicref" data-id="concept_hjv_5nn_r3b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hjv_5nn_r3b-d16893e54557-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/SQLServerBDCBulk.html#concept_hjv_5nn_r3b" id="concept_hjv_5nn_r3b-d16893e54557-link">SQL Server 2019 BDC Bulk Loader (deprecated)</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_idr_ct5_w2b-d16893e54822" class="topicref" data-id="concept_idr_ct5_w2b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_idr_ct5_w2b-d16893e54822-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Syslog.html#concept_idr_ct5_w2b" id="concept_idr_ct5_w2b-d16893e54822-link">Syslog</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hlx_r53_rx-d16893e54984" class="topicref" data-id="concept_hlx_r53_rx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hlx_r53_rx-d16893e54984-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/TableauCRM.html#concept_hlx_r53_rx" id="concept_hlx_r53_rx-d16893e54984-link">Tableau CRM</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ngm_jjf_b1c-d16893e55170" class="topicref" data-id="concept_ngm_jjf_b1c" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ngm_jjf_b1c-d16893e55170-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Teradata.html#concept_ngm_jjf_b1c" id="concept_ngm_jjf_b1c-d16893e55170-link">Teradata</a></div></div></li><li role="treeitem"><div data-tocid="concept_ryn_v3z_lr-d16893e55610" class="topicref" data-id="concept_ryn_v3z_lr" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/ToError.html" id="concept_ryn_v3z_lr-d16893e55610-link">To Error </a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem"><div data-tocid="concept_htf_ydj_wq-d16893e55634" class="topicref" data-id="concept_htf_ydj_wq" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Trash.html#concept_htf_ydj_wq" id="concept_htf_ydj_wq-d16893e55634-link">Trash</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_nt1_slb_1zb-d16893e55658" class="topicref" data-id="concept_nt1_slb_1zb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_nt1_slb_1zb-d16893e55658-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/WebClient.html#concept_nt1_slb_1zb" id="concept_nt1_slb_1zb-d16893e55658-link">Web Client</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_l4d_mjn_lz-d16893e56003" class="topicref" data-id="concept_l4d_mjn_lz" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_l4d_mjn_lz-d16893e56003-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/WebSocketClient.html#concept_l4d_mjn_lz" id="concept_l4d_mjn_lz-d16893e56003-link">WebSocket Client</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li></ul></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_umc_1lk_fx-d16893e56072" class="topicref" data-id="concept_umc_1lk_fx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_umc_1lk_fx-d16893e56072-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Executors/Executors-title.html" id="concept_umc_1lk_fx-d16893e56072-link">Executors</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xxd_f5r_kx-d16893e59696" class="topicref" data-id="concept_xxd_f5r_kx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xxd_f5r_kx-d16893e59696-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Event_Handling/EventFramework-Title.html#concept_xxd_f5r_kx" id="concept_xxd_f5r_kx-d16893e59696-link">Dataflow Triggers</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_zq5_pb4_flb-d16893e60134" class="topicref" data-id="concept_zq5_pb4_flb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_zq5_pb4_flb-d16893e60134-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/Solutions-title.html" id="concept_zq5_pb4_flb-d16893e60134-link">Solutions</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ugp_kwf_xw-d16893e61337" class="topicref" data-id="concept_ugp_kwf_xw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ugp_kwf_xw-d16893e61337-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/DPM/DPM_title.html" id="concept_ugp_kwf_xw-d16893e61337-link">StreamSets Control Hub</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fyf_gkq_4bb-d16893e62693" class="topicref" data-id="concept_fyf_gkq_4bb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fyf_gkq_4bb-d16893e62693-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Edge_Mode/EdgePipelines_title.html" id="concept_fyf_gkq_4bb-d16893e62693-link"><span class="ph">StreamSets Data Collector Edge</span></a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wwq_gxc_py-d16893e63980" class="topicref" data-id="concept_wwq_gxc_py" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wwq_gxc_py-d16893e63980-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Multithreaded_Pipelines/MultithreadedPipelines.html#concept_wwq_gxc_py" id="concept_wwq_gxc_py-d16893e63980-link">Multithreaded Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_gzw_tdm_p2b-d16893e64187" class="topicref" data-id="concept_gzw_tdm_p2b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_gzw_tdm_p2b-d16893e64187-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Microservice/Microservice_Title.html#concept_gzw_tdm_p2b" id="concept_gzw_tdm_p2b-d16893e64187-link">Microservice Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="Orchestrators_Title-d16893e64348" class="topicref" data-id="Orchestrators_Title" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action Orchestrators_Title-d16893e64348-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Orchestration_Pipelines/OrchestrationPipelines_Title.html#Orchestrators_Title" id="Orchestrators_Title-d16893e64348-link">Orchestration Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wr1_ktz_bt-d16893e64489" class="topicref" data-id="concept_wr1_ktz_bt" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wr1_ktz_bt-d16893e64489-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/RPC_Pipelines/SDC_RPCpipelines_title.html#concept_wr1_ktz_bt" id="concept_wr1_ktz_bt-d16893e64489-link">SDC RPC Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fpz_5r4_vs-d16893e64679" class="topicref" data-id="concept_fpz_5r4_vs" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fpz_5r4_vs-d16893e64679-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Cluster_Mode/ClusterPipelines_title.html" id="concept_fpz_5r4_vs-d16893e64679-link">Cluster Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jjk_23z_sq-d16893e65172" class="topicref" data-id="concept_jjk_23z_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jjk_23z_sq-d16893e65172-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Data_Preview/DataPreview_Title.html#concept_jjk_23z_sq" id="concept_jjk_23z_sq-d16893e65172-link">Data Preview</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pgk_brx_rr-d16893e65458" class="topicref" data-id="concept_pgk_brx_rr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pgk_brx_rr-d16893e65458-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Alerts/RulesAlerts_title.html#concept_pgk_brx_rr" id="concept_pgk_brx_rr-d16893e65458-link">Rules and Alerts</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_asx_fdz_sq-d16893e65960" class="topicref" data-id="concept_asx_fdz_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_asx_fdz_sq-d16893e65960-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Monitoring/PipelineMonitoring_title.html#concept_asx_fdz_sq" id="concept_asx_fdz_sq-d16893e65960-link">Pipeline Monitoring</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_o3l_dtr_5q-d16893e66304" class="topicref" data-id="concept_o3l_dtr_5q" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_o3l_dtr_5q-d16893e66304-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Maintenance/PipelineMaintenance_title.html#concept_o3l_dtr_5q" id="concept_o3l_dtr_5q-d16893e66304-link">Pipeline Maintenance</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yms_ftm_sq-d16893e66768" class="topicref" data-id="concept_yms_ftm_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yms_ftm_sq-d16893e66768-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Administration/Administration_title.html#concept_yms_ftm_sq" id="concept_yms_ftm_sq-d16893e66768-link">Administration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_nls_w1r_ks-d16893e67508" class="topicref" data-id="concept_nls_w1r_ks" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_nls_w1r_ks-d16893e67508-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/Tutorial-title.html" id="concept_nls_w1r_ks-d16893e67508-link">Tutorial</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_sh3_frm_tq-d16893e68001" class="topicref" data-id="concept_sh3_frm_tq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_sh3_frm_tq-d16893e68001-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Troubleshooting/Troubleshooting_title.html#concept_sh3_frm_tq" id="concept_sh3_frm_tq-d16893e68001-link">Troubleshooting</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xbx_rs1_tq-d16893e68798" class="topicref" data-id="concept_xbx_rs1_tq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xbx_rs1_tq-d16893e68798-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Glossary/Glossary_title.html#concept_xbx_rs1_tq" id="concept_xbx_rs1_tq-d16893e68798-link">Glossary</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jn1_nzb_kv-d16893e68843" class="topicref" data-id="concept_jn1_nzb_kv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jn1_nzb_kv-d16893e68843-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-DataFormats/DataFormat_Title.html#concept_jn1_nzb_kv" id="concept_jn1_nzb_kv-d16893e68843-link">Data Formats by Stage</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pvm_yt3_wq-d16893e68958" class="topicref" data-id="concept_pvm_yt3_wq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pvm_yt3_wq-d16893e68958-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Expression_Language/ExpressionLanguage_title.html" id="concept_pvm_yt3_wq-d16893e68958-link">Expression Language</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_vcj_1ws_js-d16893e69669" class="topicref" data-id="concept_vcj_1ws_js" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_vcj_1ws_js-d16893e69669-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-RegEx/RegEx-Title.html#concept_vcj_1ws_js" id="concept_vcj_1ws_js-d16893e69669-link">Regular Expressions</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_chv_vmj_wr-d16893e69787" class="topicref" data-id="concept_chv_vmj_wr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_chv_vmj_wr-d16893e69787-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-GrokPatterns/GrokPatterns_title.html#concept_chv_vmj_wr" id="concept_chv_vmj_wr-d16893e69787-link">Grok Patterns</a></div></div></li></ul></div>
                        

</div>
</nav>
                    


                    
                    <div id="wh_topic_body" class="col-lg-7 col-md-9 col-sm-12">
<button id="wh_close_publication_toc_button" class="close-toc-button d-none" aria-label="Toggle publishing table of content" aria-controls="wh_publication_toc" aria-expanded="true"><span class="close-toc-icon-container"><span class="close-toc-icon"></span></span></button><button id="wh_close_topic_toc_button" class="close-toc-button d-none" aria-label="Toggle topic table of content" aria-controls="wh_topic_toc" aria-expanded="true"><span class="close-toc-icon-container"><span class="close-toc-icon"></span></span></button>

                        
<div class=" wh_topic_content body "><main role="main"><article class="" role="article" aria-labelledby="ariaid-title1"><article class="nested0" aria-labelledby="ariaid-title1" id="concept_xzc_wfq_xhb">
    <h1 class="- topic/title title topictitle1" id="ariaid-title1">Azure Data Lake Storage Gen1 (deprecated)</h1>
    
    <div class="- topic/body concept/conbody body conbody"><p class="- topic/shortdesc shortdesc"></p>
        <div class="simpletable-container"><table class="- topic/simpletable simpletable frame-none" id="concept_xzc_wfq_xhb__simpletable_bc1_rxh_sgb" data-ofbid="concept_xzc_wfq_xhb__simpletable_bc1_rxh_sgb"><colgroup><col style="width:100%"/></colgroup><tbody><tr class="- topic/strow strow">
                <td class="- topic/stentry stentry"><a class="- topic/xref xref" href="../Pipeline_Configuration/ProductIcons_Doc.html#concept_mjg_ly5_pgb">Supported pipeline types:</a><ul class="- topic/ul ul" id="concept_xzc_wfq_xhb__ul_s3x_b2m_rgb" data-ofbid="concept_xzc_wfq_xhb__ul_s3x_b2m_rgb">
                        <li class="- topic/li li">
                            <p class="- topic/p p"><img class="- topic/image image" id="concept_xzc_wfq_xhb__image_t3x_b2m_rgb" src="../../../reusable-content/datacollector/reusable-topics/../../shared-graphics/icon-SDC.png" height="21" width="21"/> Data Collector</p>
                        </li>
                    </ul></td>
            </tr></tbody></table></div>
        <div class="- topic/p p">The Azure Data Lake Storage Gen1
            destination writes data to Microsoft Azure Data Lake Storage Gen1. You can use the Azure
            Data Lake Storage Gen1 destination in standalone and cluster batch pipelines. <span class="- topic/ph ph">For information about supported versions, see <a class="- topic/xref xref" href="../Installation/SupportedSystemVersions.html#concept_s5h_bcr_n4b">Supported Systems and Versions</a>.</span><div class="- topic/note note important note_important"><span class="note__title">Important:</span> <span class="- topic/ph ph" id="concept_xzc_wfq_xhb__deprecated-ph">This stage is <a class="- topic/xref xref" href="../Pipeline_Design/Deprecated.html#concept_pby_dh2_lpb">deprecated</a> and may be removed in a future release.</span></div>To write to Azure Data Lake Storage Gen2, use the <a class="- topic/xref xref" href="ADLS-G2-D.html#concept_ajp_1d2_vhb">Azure Data Lake Storage Gen2
                destination.</a>
        </div>
        <p class="- topic/p p">Before you use the destination, you must perform some prerequisite tasks.</p>
        <p class="- topic/p p">When you configure the Azure Data Lake Storage Gen1 destination, you specify information
            to connect to Azure. You define a directory template and time basis to determine the
            output directories that the destination creates and the files where records are written. </p>
        <p class="- topic/p p">You can define a file prefix and suffix, the data time zone, and
                  properties that define when the destination closes a file. You can specify the
                  amount of time that a record can be written to its associated directory and what
                  happens to late records.</p>
        <p class="- topic/p p">When desired, you can write records, use the defined Avro schema, and roll files based on
            record header attributes. For more information, see <a class="- topic/xref xref" href="../Pipeline_Design/RecordHeaderAttributes.html#concept_lmn_gdc_1w">Record Header Attributes for Record-Based Writes</a>.</p>
        <p class="- topic/p p">You can use Gzip, Bzip2, Snappy, LZ4, and other compression
                  formats to write output files. </p>
        <p class="- topic/p p"><span class="- topic/ph ph">The destination can generate events for an event stream. For
                        more information about the event framework, see <a class="- topic/xref xref" href="../Event_Handling/EventFramework-Title.html#concept_cph_5h4_lx">Dataflow Triggers Overview</a>.</span></p>
    </div>
<article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title2" id="concept_fn4_qyw_zhb">
    <h2 class="- topic/title title topictitle2" id="ariaid-title2">Prerequisites</h2>
    <div class="- topic/body concept/conbody body conbody">
        <div class="- topic/p p">Complete the following
            prerequisites before you configure the Azure Data Lake Storage Gen1 destination:<ol class="- topic/ol ol" id="concept_fn4_qyw_zhb__ol_xrz_g1l_vhb" data-ofbid="concept_fn4_qyw_zhb__ol_xrz_g1l_vhb">
                <li class="- topic/li li">If necessary, create a new Azure Active Directory
                    application for <span class="- topic/ph ph">Data Collector</span>.<p class="- topic/p p" id="concept_fn4_qyw_zhb__p_ADLS-Auth-CreateAppInfo" data-ofbid="concept_fn4_qyw_zhb__p_ADLS-Auth-CreateAppInfo">For information about creating a new
                        application, see the <a class="- topic/xref xref" href="https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal" target="_blank" rel="external noopener">Azure documentation</a>.</p></li>
                <li class="- topic/li li"><span class="- topic/ph ph">Ensure that the Azure Active Directory <span class="- topic/ph ph">Data Collector</span> application
                        has the appropriate access control to perform the necessary
                        tasks.</span><p class="- topic/p p">The <span class="- topic/ph ph">Data Collector</span> application
                        requires Write and Execute permissions to write data to Azure. If also
                        reading from Azure, the application requires Read permission as well.
                            </p><p class="- topic/p p">For information about configuring Gen1
                                    access control, see the <a class="- topic/xref xref" href="https://docs.microsoft.com/en-us/azure/data-lake-store/data-lake-store-security-overview" target="_blank" rel="external noopener">Azure documentation</a>.
                              </p></li>
                <li class="- topic/li li" id="concept_fn4_qyw_zhb__Azure-Auth-StorageAcct" data-ofbid="concept_fn4_qyw_zhb__Azure-Auth-StorageAcct">If necessary, create an Azure Data Lake Storage Gen1
                    storage account. <p class="- topic/p p">For information about creating an account, see the <a class="- topic/xref xref" href="https://docs.microsoft.com/en-us/azure/data-lake-store/data-lake-store-get-started-portal#create-a-data-lake-storage-gen1-account" target="_blank" rel="external noopener">Azure documentation</a>.</p></li>
                <li class="- topic/li li">Create a folder in the storage account. The destination writes data to the
                    folder.  <p dir="ltr" class="- topic/p p">For information about creating a folder, see the <a class="- topic/xref xref" href="https://docs.microsoft.com/en-us/azure/data-lake-store/data-lake-store-get-started-portal#createfolder" target="_blank" rel="external noopener">Azure documentation</a>.</p></li>
                <li class="- topic/li li"><a class="- topic/xref xref" href="ADLS-G1-D.html#concept_jwr_vyw_zhb">Retrieve
                        information from Azure</a> to configure the destination. </li>
            </ol></div>
        <p class="- topic/p p">After you complete all of the prerequisite tasks, you can configure the Azure Data Lake
            Storage Gen1 destination. </p>
    </div>
<article class="- topic/topic concept/concept topic concept nested2" aria-labelledby="ariaid-title3" id="concept_jwr_vyw_zhb">
    <h3 class="- topic/title title topictitle3" id="ariaid-title3">Retrieve Authentication Information</h3>
    <div class="- topic/body concept/conbody body conbody">
        <p class="- topic/p p">Azure Data Lake Storage Gen1 destination
            connects to Azure using Azure Active Directory service principal authentication, also
            known as service-to-service authentication. </p>
        <div class="- topic/p p">The destination requires the following Azure authentication information: <ul class="- topic/ul ul" id="concept_jwr_vyw_zhb__ul_rg1_dgx_xhb" data-ofbid="concept_jwr_vyw_zhb__ul_rg1_dgx_xhb">
            <li class="- topic/li li">Application ID - <span class="- topic/ph ph" id="concept_jwr_vyw_zhb__Azure-AppID-def">Application ID for the Azure Active Directory <span class="- topic/ph ph">Data Collector</span>
                                    application. Also known as the client ID. </span><p class="- topic/p p" id="concept_jwr_vyw_zhb__Azure-GetKeyandKey" data-ofbid="concept_jwr_vyw_zhb__Azure-GetKeyandKey">For information on accessing the
                                    application ID from the Azure portal, see the <a class="- topic/xref xref" href="https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#get-application-id-and-authentication-key" target="_blank" rel="external noopener">Azure
                                    documentation</a>.</p></li>
            <li class="- topic/li li">Auth Token Endpoint - <span class="- topic/ph ph" id="concept_jwr_vyw_zhb__Azure-Gen2-AuthToken-def">OAuth 2.0 token endpoint for
                    the Azure Active Directory v1.0 application for <span class="- topic/ph ph">Data Collector</span>. For example:
                        <code class="+ topic/ph pr-d/codeph ph codeph">https://login.microsoftonline.com/&lt;uuid&gt;/oauth2/token.</code></span>
            </li>
            <li class="- topic/li li">Application Key - <span class="- topic/ph ph" id="concept_jwr_vyw_zhb__Azure-Gen2-AuthKey-def">Authentication key or client secret
                                    for the Azure Active Directory application. Also known as the
                                    client secret.</span><p class="- topic/p p" id="concept_jwr_vyw_zhb__Azure-GetKeyandKey2" data-ofbid="concept_jwr_vyw_zhb__Azure-GetKeyandKey2">For information
                                    on accessing the application key from the Azure portal, see the
                                        <a class="- topic/xref xref" href="https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#get-application-id-and-authentication-key" target="_blank" rel="external noopener">Azure
                                    documentation</a>.</p></li>
        </ul></div>
    </div>
</article></article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title4" id="concept_e5y_2zw_zhb">
    <h2 class="- topic/title title topictitle2" id="ariaid-title4">Directory Templates</h2>
    <div class="- topic/body concept/conbody body conbody">
        <p class="- topic/p p">By default, the Azure
            Data Lake Storage Gen1 destination uses directory templates to create output
            directories. The destination writes records to the directories based on the configured
            time basis. </p>
        <p class="- topic/p p">You can alternatively write records to directories based
                  on the <code class="+ topic/ph pr-d/codeph ph codeph">targetDirectory</code> record header attribute. Using the
                        <code class="+ topic/ph pr-d/codeph ph codeph">targetDirectory</code> attribute disables the ability to define
                  directory templates.</p>
        <p class="- topic/p p">When you define a directory template, you can use a mix
                  of constants, field values, and datetime variables. You can use the
                        <code class="+ topic/ph pr-d/codeph ph codeph">every</code> function to create new directories at regular
                  intervals based on hours, minutes, or seconds, starting on the hour. You can also
                  use the <code class="+ topic/ph pr-d/codeph ph codeph">record:valueOrDefault</code> function to create new directories
                  from field values or a default in the directory template. </p>
        <div class="- topic/p p">For example, the following directory template creates
                  output directories for event data based on the state and timestamp of a record
                  with hours as the smallest unit of measure, creating a new directory every
                  hour:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code> /outputfiles/${record:valueOrDefault("/State", "unknown")}/${YY()}-${MM()}-${DD()}-${hh()}</code></pre></div>
        <div class="- topic/p p">You can use the following elements in a directory template:<dl class="- topic/dl dl">
                        
                              <dt class="- topic/dt dt dlterm">Constants</dt>
                              <dd class="- topic/dd dd">You can use any constant, such as <code class="+ topic/ph pr-d/codeph ph codeph">output</code>.</dd>
                        
                        
                              <dt class="- topic/dt dt dlterm">Datetime Variables</dt>
                              <dd class="- topic/dd dd">You can use datetime variables, such as <code class="+ topic/ph pr-d/codeph ph codeph">${YYYY()}</code>
                                    or <code class="+ topic/ph pr-d/codeph ph codeph">${DD()}</code>. The destination creates directories
                                    as needed, based on the smallest datetime variable that you use.
                                    For example, if the smallest variable is hours, then the
                                    directories are created for every hour of the day that receives
                                    output records.</dd>
                              <dd class="- topic/dd dd ddexpand">When you use datetime variables in an expression, use all of the
                                    datetime variables between one of the year variables and the
                                    smallest variable that you want to use. Do not skip a variable
                                    within the progression. For example, to create directories on a
                                    daily basis, use a year variable, a month variable, and then a
                                    day variable. You might use one of the following datetime
                                    variable progressions: </dd>
                              <dd class="- topic/dd dd ddexpand">
                                    <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>${YYYY()}-${MM()}-${DD()}
${YY()}_${MM()}_${DD()}</code></pre>
                              </dd>
                              <dd class="- topic/dd dd ddexpand">For details about datetime variables, see <a class="- topic/xref xref" href="../Expression_Language/DateTimeVariables.html#concept_gh4_qd2_sv" title="The expression language provides datetime variables for use in expressions.">Datetime Variables</a>.</dd>
                        
                        
                              <dt class="- topic/dt dt dlterm"><code class="+ topic/ph pr-d/codeph ph codeph">every</code> function</dt>
                              <dd class="- topic/dd dd">You can use the <code class="+ topic/ph pr-d/codeph ph codeph">every</code> function in a directory
                                    template to create directories at regular intervals based on
                                    hours, minutes, or seconds, beginning on the hour. The intervals
                                    should be a submultiple or integer factor of 60. For example,
                                    you can create directories every 15 minutes or 30 seconds. </dd>
                              <dd class="- topic/dd dd ddexpand">Use the <code class="+ topic/ph pr-d/codeph ph codeph">every</code> function to replace the smallest
                                    datetime variable used in the template.</dd>
                              <dd class="- topic/dd dd ddexpand">For example, the following directory template creates directories
                                    every 5 minutes, starting on the
                                    hour:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>/HDFS_output/${YYYY()}-${MM()}-${DD()}-${hh()}-${every(5,mm())}</code></pre></dd>
                              <dd class="- topic/dd dd ddexpand">For details about the <code class="+ topic/ph pr-d/codeph ph codeph">every</code> function, see <a class="- topic/xref xref" href="../Expression_Language/Functions.html#concept_ddw_ld1_1s">Miscellaneous Functions</a>.</dd>
                        
                        
                              <dt class="- topic/dt dt dlterm"><code class="+ topic/ph pr-d/codeph ph codeph">record:valueOrDefault</code> function</dt>
                              <dd class="- topic/dd dd">You can use the <code class="+ topic/ph pr-d/codeph ph codeph">record:valueOrDefault</code> function in
                                    a directory template to create directories with the value of a
                                    field or the specified default value if the field does not exist
                                    or if the field is null:
                                    <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>${record:valueOrDefault(&lt;field path&gt;, &lt;default value&gt;)}</code></pre></dd>
                              <dd class="- topic/dd dd ddexpand">For example, the following directory template creates a directory
                                    based on the <code class="+ topic/ph pr-d/codeph ph codeph">Product</code> field every day, and if
                                    the <code class="+ topic/ph pr-d/codeph ph codeph">Product</code> field is empty or null, uses
                                          <code class="+ topic/ph pr-d/codeph ph codeph">Misc</code> in the directory path:
                                    <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>/${record:valueOrDefault("/Product", "Misc")}/${YY()}-${MM()}-${DD()}</code></pre></dd>
                              <dd class="- topic/dd dd ddexpand">This template might create the following
                                    paths:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>/Shirts/2015-07-31 
/Misc/2015-07-31</code></pre></dd>
                        
                  </dl></div>
    </div>
</article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title5" id="concept_wps_x2x_zhb">
    <h2 class="- topic/title title topictitle2" id="ariaid-title5">Time Basis</h2>
    <div class="- topic/body concept/conbody body conbody">
        <p class="- topic/p p">When using directory templates, the time
            basis helps determine when directories are created. It also determines the directory
            that the destination uses when writing a record, and whether a record is late.</p>
        <div class="- topic/p p">You can use the following times as the time basis: <dl class="- topic/dl dl">
                  
                        <dt class="- topic/dt dt dlterm">Processing Time</dt>
                        <dd class="- topic/dd dd">When you use processing time as the time basis, the destination creates
                              directories based on the processing time and the directory template,
                              and writes records to the directories based on when they are
                              processed.</dd>
                        <dd class="- topic/dd dd ddexpand">For example, say a directory template creates directories every minute
                              and the time basis is the time of processing. Then, directories are
                              created for every minute that the destination writes output records.
                              And the output records are written to the directory for that minute of
                              processing. </dd>
                        <dd class="- topic/dd dd ddexpand">To use the processing time as the time basis, use the following
                              expression: <code class="+ topic/ph pr-d/codeph ph codeph">${time:now()}</code>. This is the default time
                              basis. </dd>
                  
                  
                        <dt class="- topic/dt dt dlterm">Record Time</dt>
                        <dd class="- topic/dd dd">When you use the time associated with a record as the time basis, you
                              specify a Date field in the record. The destination creates
                              directories based on the datetimes associated with the records and
                              writes the records to the appropriate directories. </dd>
                        <dd class="- topic/dd dd ddexpand">For example, say a directory template creates directories every hour and
                              the time basis is based on the record. Then, directories are created
                              for every hour associated with output records and the destination
                              writes the records to the related output directory. </dd>
                        <dd class="- topic/dd dd ddexpand">To use a time associated with the record, use an expression that calls a
                              field and resolves to a datetime value, such as
                                    <code class="+ topic/ph pr-d/codeph ph codeph">${record:value("/Timestamp")}</code>. </dd>
                  
            </dl></div>
    </div>
</article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title6" id="concept_x35_tgx_zhb">
    <h2 class="- topic/title title topictitle2" id="ariaid-title6">Late Records and Late Record Handling</h2>
    <div class="- topic/body concept/conbody body conbody">
        <p class="- topic/p p"></p>
        <p class="- topic/p p">When you use a record time as the time basis, you can define
                  a time limit for records to be written to their associated output file. When the
                  destination creates a new output file in a new directory, the previous output file
                  is kept open for the specified late record time limit. When records that belong in
                  that file arrive within the time limit, the destination writes the records to the
                  open output file. When the late record time limit is reached, the output file is
                  closed and any record that arrives past this limit is considered late.</p>
        <div class="- topic/note note tip note_tip"><span class="note__title">Tip:</span> The late records properties are not applicable if
                  you use processing time as the time basis. If you use processing time, set the
                  late record time limit to one second.</div>
        <p class="- topic/p p">You can send late records to a late records file or to the
                  stage for error handling. When you send records to a late records file, you define
                  a late records directory template. </p>
        <div class="- topic/p p">For example, you use a record time as the time basis,
                  configure a one-hour late record time limit, configure late records to be sent to
                  the stage for error handling, and use the default directory template value:
                  <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>/tmp/out/${YYYY()}-${MM()}-${DD()}-${hh()} </code></pre></div>
        <p class="- topic/p p">The first records that arrive have a datetime between the
                  hours of 02:00 and 02:59, and so are written to an output file in the 02
                  directory. When records with a datetime between the hours of 03:00 and 03:59
                  arrive, the destination creates a new file in an 03 directory. The destination
                  keeps the file in the 02 directory open for another hour. </p>
        <p class="- topic/p p">If a record with a datetime between the hours of 02:00 and
                  02:59 arrives before the hour time limit, the destination writes the record to the
                  open file in the 02 directory. After one hour, the destination closes the output
                  file in the 02 directory. Any records with a datetime between the hours of 02:00
                  and 02:59 that arrive after the one-hour time limit are considered late. The late
                  records are sent to the stage for error handling.</p>
    </div>
</article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title7" id="concept_kyy_chx_zhb">
    <h2 class="- topic/title title topictitle2" id="ariaid-title7">Timeout to Close Idle Files</h2>
    <div class="- topic/body concept/conbody body conbody">
        <p class="- topic/p p">You can configure the maximum
            time that an open output file can remain idle. After no records are written to an output
            file for the specified amount of time, the destination closes the file. </p>
        <p class="- topic/p p">You might want to configure an idle timeout when output
                  files remain open and idle for too long, thus delaying another system from
                  processing the files.</p>
        <div class="- topic/p p">Output files might remain idle for too long for the following
                  reasons: <ul class="- topic/ul ul" id="concept_kyy_chx_zhb__ul_pcq_5l3_mw" data-ofbid="concept_kyy_chx_zhb__ul_pcq_5l3_mw">
                        <li class="- topic/li li">You configured the maximum number of records to be written to output
                              files or the maximum size of output files, but records have stopped
                              arriving. An output file that has not reached the maximum number of
                              records or the maximum file size stays open until more records
                              arrive.</li>
                        <li class="- topic/li li">You configured a date field in the record as the time basis and have
                              configured a late record time limit, but records arrive in
                              chronological order. When a new directory is created, the output file
                              in the previous directory remains open for the configured late record
                              time limit. However, no records are ever written to the open file in
                              the previous directory.<p class="- topic/p p">For example, when a record with a datetime
                                    of 03:00 arrives, the destination creates a new file in a new 03
                                    directory. The previous file in the 02 directory is kept open
                                    for the late record time limit, which is an hour by default.
                                    However, when records arrive in chronological order, no records
                                    that belong in the 02 directory arrive after the 03 directory is
                                    created. </p></li>
                  </ul></div>
        <p class="- topic/p p">In either situation, configure an idle timeout so that other
                  systems can process the files sooner, instead of waiting for the configured
                  maximum records, maximum file size, or late records conditions to occur. </p>
    </div>
</article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title8" id="concept_uyx_3bt_13b">
 <h2 class="- topic/title title topictitle2" id="ariaid-title8">Recovery</h2>
 <div class="- topic/body concept/conbody body conbody">
  <p class="- topic/p p">The Azure Data Lake
            Storage Gen1 destination supports recovery after an unexpected stop of the pipeline by
            renaming temporary files when the pipeline restarts.</p>
        <div class="- topic/p p">The destination names temporary open output files using the
                  following format:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>_tmp_&lt;prefix&gt;_&lt;runnerId&gt;</code></pre>Where
                        <code class="+ topic/ph pr-d/codeph ph codeph">&lt;prefix&gt;</code> is the file prefix defined for the destination
                  and <code class="+ topic/ph pr-d/codeph ph codeph">&lt;runnerId&gt;</code> is the ID of the pipeline runner performing the
                  pipeline processing. For example, when the destination prefix is defined as
                        <code class="+ topic/ph pr-d/codeph ph codeph">sdc</code> and the destination runs from a single-threaded
                  pipeline, the temporary file is named like so: <code class="+ topic/ph pr-d/codeph ph codeph">_tmp_sdc_0</code>. </div><div class="- topic/p p">When the destination closes the file, either after it is fully
                  written, after the idle timeout expires, or when you deliberately stop the
                  pipeline, it renames the file to remove the <code class="+ topic/ph pr-d/codeph ph codeph">_tmp_</code> string and to
                  replace the pipeline runner ID with a random unique identifier like
                  so:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>&lt;prefix&gt;_e7ce67c5-013d-47a7-9496-8c882ddb28a0</code></pre></div><p class="- topic/p p">However, when the pipeline stops unexpectedly, the temporary
                  files remain. When the pipeline restarts, the destination scans all subdirectories
                  of the defined directory template to rename any temporary files that match the
                  defined prefix for the destination. After the destination renames the temporary
                  files, it continues writing to new output files.</p><div class="- topic/note note note note_note"><span class="note__title">Note:</span> The destination renames all temporary files that match the
                  defined prefix in all subdirectories of the defined directory template, even if
                  those files were not written by that pipeline. So if you happen to have another
                  file whose name begins with the same pattern - <code class="+ topic/ph pr-d/codeph ph codeph">_tmp_&lt;prefix&gt;</code> -
                  the destination renames that file also.</div><div class="- topic/p p">The destination might not rename all temporary files in the
                  following situations:<dl class="- topic/dl dl">
                        
                              <dt class="- topic/dt dt dlterm">The directory template includes an expression with the
                                    record:value or record:valueOrDefault function.</dt>
                              <dd class="- topic/dd dd">If the record:value or record:valueOrDefault function evaluates to
                                    an empty string or to a subdirectory, the destination cannot
                                    determine those locations when the pipeline restarts. As a
                                    result, the destination cannot rename any temporary files
                                    written to those locations.</dd>
                              <dd class="- topic/dd dd ddexpand">For example, letâs assume that the directory template is defined
                                    as
                                    follows:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>/tmp/out/${YY()}-${MM()}-${DD()}/${sdc:hostname()}/${record:value('/a')}/${record:value('/b')}</code></pre></dd>
                              <dd class="- topic/dd dd ddexpand">If the expression <code class="+ topic/ph pr-d/codeph ph codeph">${record:value('/b')}</code> evaluates
                                    to an empty string or to a subdirectory such as
                                          <code class="+ topic/ph pr-d/codeph ph codeph">/folder1/folder2</code>, then the destination
                                    cannot determine those locations when the pipeline restarts.
                              </dd>
                        
                        
                              <dt class="- topic/dt dt dlterm">The directory is defined in the targetDirectory record header
                                    attribute.</dt>
                              <dd class="- topic/dd dd">When the directory is defined in the targetDirectory record header
                                    attribute, the destination cannot determine where to look for
                                    temporary files when the pipeline restarts. As a result, it
                                    cannot rename the temporary files.</dd>
                        
                  </dl></div><p class="- topic/p p">In either of these situations, you must manually rename the
                  temporary files.</p><p class="- topic/p p">File recovery can slow down the pipeline as it restarts. If
                  needed, you can configure the destination to skip file recovery.</p>
 </div>
</article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title9" id="concept_et1_lhx_zhb">
    <h2 class="- topic/title title topictitle2" id="ariaid-title9">Event Generation</h2>
    <div class="- topic/body concept/conbody body conbody">
        <p class="- topic/p p">The
            Azure Data Lake Storage Gen1 destination can generate events that you can use in an
            event stream. When you enable event generation, the Azure Data Lake Storage Gen1
            destination generates event records each time the destination completes writing to an
            output file or completes streaming a whole file. </p>
        <div class="- topic/p p">The events can be used in any logical way. For example: <ul class="- topic/ul ul" id="concept_et1_lhx_zhb__ul_r5n_3sk_vhb" data-ofbid="concept_et1_lhx_zhb__ul_r5n_3sk_vhb">
                <li class="- topic/li li">With an ADLS Gen1 File Metadata executor to change file metadata, create an
                    empty file, or remove a file or directory after closing a file.<p class="- topic/p p">For an example
                        of a similar use case, see <a class="- topic/xref xref" href="../Solutions/FileManagement.html#concept_d1q_xl4_lx">Managing Output Files</a>.</p></li>
                <li class="- topic/li li">With the Email executor to send a custom email
                              after receiving an event.<p class="- topic/p p">For an example, see <a class="- topic/xref xref" href="../Solutions/SendEmail.html#concept_t2t_lp5_xz">Sending Email During Pipeline Processing</a>.</p></li>
                <li class="- topic/li li">With a destination to store event information.
                                    <p class="- topic/p p">For an example, see <a class="- topic/xref xref" href="../Solutions/EventStorage.html#concept_ocb_nnl_px">Preserving an Audit Trail of Events</a>.</p></li>
            </ul></div>
        <p class="- topic/p p"><span class="- topic/ph ph">For more information about dataflow
                        triggers and the event framework, see <a class="- topic/xref xref" href="../Event_Handling/EventFramework-Title.html#concept_cph_5h4_lx">Dataflow Triggers Overview</a>.</span></p>
    </div>
<article class="- topic/topic concept/concept topic concept nested2" aria-labelledby="ariaid-title10" id="concept_concept_aps_qhx_zhb">
    <h3 class="- topic/title title topictitle3" id="ariaid-title10">Event Records</h3>
    <div class="- topic/body concept/conbody body conbody">
        <p class="- topic/p p">The Azure Data Lake Storage Gen1
            destination event records include the following event-related record header attributes.
            Record header attributes are stored as String values.</p>
        <div class="- topic/p p">
            <div class="table-container"><table class="- topic/table table frame-all" id="concept_concept_aps_qhx_zhb__table_ilm_xsc_rx" data-ofbid="concept_concept_aps_qhx_zhb__table_ilm_xsc_rx" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
            <tr class="- topic/row">
              <th class="- topic/entry entry colsep-1 rowsep-1" id="concept_concept_aps_qhx_zhb__table_ilm_xsc_rx__entry__1">Record Header Attribute</th>
              <th class="- topic/entry entry colsep-0 rowsep-1" id="concept_concept_aps_qhx_zhb__table_ilm_xsc_rx__entry__2">Description</th>
            </tr>
          </thead><tbody class="- topic/tbody tbody">
            <tr class="- topic/row">
              <td class="- topic/entry entry colsep-1 rowsep-1" id="concept_concept_aps_qhx_zhb__entry-eventType" headers="concept_concept_aps_qhx_zhb__table_ilm_xsc_rx__entry__1">sdc.event.type</td>
              <td class="- topic/entry entry colsep-0 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_ilm_xsc_rx__entry__2">Event type. Uses one of the following types:<ul class="- topic/ul ul" id="concept_concept_aps_qhx_zhb__ul_m12_mgp_qx" data-ofbid="concept_concept_aps_qhx_zhb__ul_m12_mgp_qx">
                  <li class="- topic/li li" id="concept_concept_aps_qhx_zhb__event-file-closed" data-ofbid="concept_concept_aps_qhx_zhb__event-file-closed">file-closed - Generated when the destination closes a
                    file.</li>
                  <li class="- topic/li li">wholeFileProcessed - Generated when the destination completes streaming a
                    whole file.</li>
                </ul></td>
            </tr>
            <tr class="- topic/row" id="concept_concept_aps_qhx_zhb__row-eventVersion">
              <td class="- topic/entry entry colsep-1 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_ilm_xsc_rx__entry__1">sdc.event.version</td>
              <td class="- topic/entry entry colsep-0 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_ilm_xsc_rx__entry__2">Integer that indicates the version of the event record type.</td>
            </tr>
            <tr class="- topic/row" id="concept_concept_aps_qhx_zhb__row-eventTimestamp">
              <td class="- topic/entry entry colsep-1 rowsep-0" headers="concept_concept_aps_qhx_zhb__table_ilm_xsc_rx__entry__1">sdc.event.creation_timestamp</td>
              <td class="- topic/entry entry colsep-0 rowsep-0" id="concept_concept_aps_qhx_zhb__entry-Event-creationDate" headers="concept_concept_aps_qhx_zhb__table_ilm_xsc_rx__entry__2">Epoch timestamp when the stage created the event.
              </td>
            </tr>
          </tbody></table></div>
        </div>
        <div class="- topic/p p"><span class="- topic/ph ph" id="concept_concept_aps_qhx_zhb__ph-EventRecordsStemSentence">The destination can
                        generate the following types of event records:</span><dl class="- topic/dl dl">
                        
                              <dt class="- topic/dt dt dlterm" id="concept_concept_aps_qhx_zhb__dlentry-FileClosure-StageEventRecords" data-ofbid="concept_concept_aps_qhx_zhb__dlentry-FileClosure-StageEventRecords">File closure</dt>
                              <dd class="- topic/dd dd">The destination generates a file closure event record when it
                                    closes an output file. </dd>
                              <dd class="- topic/dd dd ddexpand">File closure event records have the
                                          <code class="+ topic/ph pr-d/codeph ph codeph">sdc.event.type</code> record header attribute
                                    set to <code class="+ topic/ph pr-d/codeph ph codeph">file-closed</code> and include the following
                                          fields:<div class="table-container"><table class="- topic/table table frame-all" id="concept_concept_aps_qhx_zhb__table_dh2_5gc_rx" data-ofbid="concept_concept_aps_qhx_zhb__table_dh2_5gc_rx" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                                  <tr class="- topic/row">
                                                  <th class="- topic/entry entry colsep-1 rowsep-1" id="concept_concept_aps_qhx_zhb__table_dh2_5gc_rx__entry__1">Field</th>
                                                  <th class="- topic/entry entry colsep-0 rowsep-1" id="concept_concept_aps_qhx_zhb__table_dh2_5gc_rx__entry__2">Description</th>
                                                  </tr>
                                                </thead><tbody class="- topic/tbody tbody">
                                                  <tr class="- topic/row">
                                                  <td class="- topic/entry entry colsep-1 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_dh2_5gc_rx__entry__1">filepath</td>
                                                  <td class="- topic/entry entry colsep-0 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_dh2_5gc_rx__entry__2">Absolute path to the closed file. </td>
                                                  </tr>
                                                  <tr class="- topic/row">
                                                  <td class="- topic/entry entry colsep-1 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_dh2_5gc_rx__entry__1">filename</td>
                                                  <td class="- topic/entry entry colsep-0 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_dh2_5gc_rx__entry__2">File name of the closed file.</td>
                                                  </tr>
                                                  <tr class="- topic/row">
                                                  <td class="- topic/entry entry colsep-1 rowsep-0" headers="concept_concept_aps_qhx_zhb__table_dh2_5gc_rx__entry__1">length</td>
                                                  <td class="- topic/entry entry colsep-0 rowsep-0" headers="concept_concept_aps_qhx_zhb__table_dh2_5gc_rx__entry__2">Size of the closed file in bytes.</td>
                                                  </tr>
                                                </tbody></table></div></dd>
                        
                        
                              <dt class="- topic/dt dt dlterm" id="concept_concept_aps_qhx_zhb__dt-WholeFileProcessed-StageEventRecords" data-ofbid="concept_concept_aps_qhx_zhb__dt-WholeFileProcessed-StageEventRecords">Whole file
                                    processed</dt>
                              <dd class="- topic/dd dd"><span class="- topic/ph ph" id="concept_concept_aps_qhx_zhb__ph-dd-WholeFileProcessed-StageEventRecords">The
                                          destination generates an event record when it completes
                                          streaming a whole file. Whole file event records have the
                                                <code class="+ topic/ph pr-d/codeph ph codeph">sdc.event.type</code> record header
                                          attribute set to <code class="+ topic/ph pr-d/codeph ph codeph">wholeFileProcessed</code> and
                                          have the following fields:</span><div class="table-container"><table class="- topic/table table frame-all" id="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx" data-ofbid="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                                  <tr class="- topic/row">
                                                  <th class="- topic/entry entry colsep-1 rowsep-1" id="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx__entry__1">Field</th>
                                                  <th class="- topic/entry entry colsep-0 rowsep-1" id="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx__entry__2">Description</th>
                                                  </tr>
                                                </thead><tbody class="- topic/tbody tbody">
                                                  <tr class="- topic/row" id="concept_concept_aps_qhx_zhb__row-sourceFileInfo-WholeFile-StageEventRecords">
                                                  <td class="- topic/entry entry colsep-1 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx__entry__1">sourceFileInfo</td>
                                                  <td class="- topic/entry entry colsep-0 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx__entry__2">A map of attributes about the original
                                                  whole file that was processed. The attributes
                                                  include:<ul class="- topic/ul ul" id="concept_concept_aps_qhx_zhb__ul_nbd_cmc_rx" data-ofbid="concept_concept_aps_qhx_zhb__ul_nbd_cmc_rx">
                                                  <li class="- topic/li li">size - Size of the whole file in bytes. </li>
                                                  </ul><p class="- topic/p p">Additional attributes depend on the
                                                  information provided by the origin system.
                                                  </p></td>
                                                  </tr>
                                                  <tr class="- topic/row" id="concept_concept_aps_qhx_zhb__row-targetFileInfo-WholeFile-StageEventRecords">
                                                  <td class="- topic/entry entry colsep-1 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx__entry__1">targetFileInfo</td>
                                                  <td class="- topic/entry entry colsep-0 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx__entry__2">A map of attributes about the whole file
                                                  written to the destination. The attributes
                                                  include:<ul class="- topic/ul ul" id="concept_concept_aps_qhx_zhb__ul_kx1_klc_rx" data-ofbid="concept_concept_aps_qhx_zhb__ul_kx1_klc_rx">
                                                  <li class="- topic/li li">path - An absolute path to the processed whole
                                                  file.</li>
                                                  </ul></td>
                                                  </tr>
                                                  <tr class="- topic/row" id="concept_concept_aps_qhx_zhb__row-D-Event-Checksum">
                                                  <td class="- topic/entry entry colsep-1 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx__entry__1">checksum</td>
                                                  <td class="- topic/entry entry colsep-0 rowsep-1" headers="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx__entry__2">Checksum generated for the written file.
                                                  <p class="- topic/p p">Included only when you configure the
                                                  destination to include checksums in the event
                                                  record. </p></td>
                                                  </tr>
                                                  <tr class="- topic/row" id="concept_concept_aps_qhx_zhb__row-D-Event-CheckAlgo">
                                                  <td class="- topic/entry entry colsep-1 rowsep-0" headers="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx__entry__1">checksumAlgorithm</td>
                                                  <td class="- topic/entry entry colsep-0 rowsep-0" headers="concept_concept_aps_qhx_zhb__table_eh2_5gc_rx__entry__2">Algorithm used to generate the checksum.
                                                  <p class="- topic/p p">Included only when you configure the
                                                  destination to include checksums in the event
                                                  record. </p></td>
                                                  </tr>
                                                </tbody></table></div></dd>
                        
                  </dl></div>
    </div>
</article></article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title11" id="concept_ymd_3fz_j4b">
    <h2 class="- topic/title title topictitle2" id="ariaid-title11">Resolving Out of Memory Errors</h2>
    <div class="- topic/body concept/conbody body conbody">
        <p class="- topic/p p">The Azure Data Lake Storage Gen1 <span class="- topic/ph ph">destination can encounter out of memory errors when the available
                        memory is not sufficient for writing output files. Typically, this occurs
                        when writing large files, but can occur with smaller files when sufficient
                        memory is not available. This is due to a known issue with Hadoop.</span></p>
        <div class="- topic/p p">If the destination encounters out of memory errors, you can resolve
                  the issue by setting the Azure configuration property,
                        <code class="+ topic/ph pr-d/codeph ph codeph">fs.azure.write.request.size</code>, as follows:<ol class="- topic/ol ol" id="concept_ymd_3fz_j4b__ol_ayh_ttz_j4b" data-ofbid="concept_ymd_3fz_j4b__ol_ayh_ttz_j4b">
                        <li class="- topic/li li">On the Delta Lake tab of the destination, add
                                    <code class="+ topic/ph pr-d/codeph ph codeph">fs.azure.write.request.size</code> to the Additional
                              Configuration property.</li>
                        <li class="- topic/li li">Determine the maximum amount of memory that you want the destination to
                              use, then divide that by a number based on the cores on the <span class="- topic/ph ph">Data Collector</span>
                              machine, as follows:
                                    <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>&lt;max destination memory&gt; / (12 * &lt;<span class="- topic/ph ph">Data Collector</span> cores&gt;)</code></pre><div class="- topic/p p">If
                                    the destination is in a multithreaded pipeline, then you must
                                    also divide by the number of threads used by the pipeline, as
                                    follows:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>(&lt;max destination memory&gt; / &lt;number of threads&gt;) / (12 * &lt;<span class="- topic/ph ph">Data Collector</span> cores&gt;) </code></pre></div></li>
                        <li class="- topic/li li">Set the <code class="+ topic/ph pr-d/codeph ph codeph">fs.azure.write.request.size</code> property to the
                              results of your calculation.</li>
                  </ol></div>
        <p class="- topic/p p">For more information about the Hadoop issue, HADOOP-16854, see
                  the <a class="- topic/xref xref" href="https://issues.apache.org/jira/browse/HADOOP-16854" target="_blank" rel="external noopener">Hadoop JIRA project</a>.</p>
    </div>
</article><article class="- topic/topic concept/concept topic concept nested1" aria-labelledby="ariaid-title12" id="concept_wld_zhx_zhb">
    <h2 class="- topic/title title topictitle2" id="ariaid-title12">Data Formats</h2>
    <div class="- topic/body concept/conbody body conbody">
        <div class="- topic/p p">The Azure Data Lake Storage Gen1
            destination writes data to Azure Data Lake Storage Gen1 based on the data format that
            you select. You can use the following data formats: <dl class="- topic/dl dl">
                
                              <dt class="- topic/dt dt dlterm">Avro</dt>
                              <dd class="- topic/dd dd">The destination writes records based on the Avro schema. You can
                                    use one of the following methods to specify the location of the
                                    Avro schema definition:</dd>
                              <dd class="- topic/dd dd ddexpand">
                                    <ul class="- topic/ul ul" id="concept_wld_zhx_zhb__ul_iyw_vrf_lx" data-ofbid="concept_wld_zhx_zhb__ul_iyw_vrf_lx">
                                          <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">In Pipeline Configuration</span> - Use
                                                the schema that you provide in the stage
                                                configuration.</li>
                                          <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">In Record Header</span> - Use the
                                                schema included in the avroSchema record header
                                                attribute.</li>
                                          <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Confluent Schema Registry</span> -
                                                Retrieve the schema from Confluent Schema Registry.
                                                Confluent Schema Registry is a distributed storage
                                                layer for Avro schemas. You can configure the
                                                destination to look up the schema in Confluent
                                                Schema Registry by the schema ID or subject. <p class="- topic/p p">If
                                                  using the Avro schema in the stage or in the
                                                  record header attribute, you can optionally
                                                  configure the destination to register the Avro
                                                  schema with Confluent Schema Registry.</p></li>
                                    </ul>
                              </dd>
                              <dd class="- topic/dd dd ddexpand">The destination includes the schema definition in each file.</dd>
                              <dd class="- topic/dd dd ddexpand">You can compress data with an Avro-supported compression codec.
                                    When using Avro compression, avoid using other compression
                                    properties in the destination. </dd>
                        
                
                              <dt class="- topic/dt dt dlterm">Binary</dt>
                              <dd class="- topic/dd dd">The stage writes <span class="- topic/ph ph" id="concept_wld_zhx_zhb__Write-Binary-ph">binary data to a single
                                          field in the record. </span></dd>
                        
                
                              <dt class="- topic/dt dt dlterm">Delimited</dt>
                              <dd class="- topic/dd dd">The destination writes <span class="- topic/ph ph" id="concept_wld_zhx_zhb__WriteDelim-InfoPh">records as
                                          delimited data. When you use this data format, the root
                                          field must be list or list-map.</span></dd>
                              <dd class="- topic/dd dd ddexpand">
                                    <div class="- topic/p p">You can use the following delimited format types: <ul class="- topic/ul ul" id="concept_wld_zhx_zhb__ul_jtv_5sl_mcb" data-ofbid="concept_wld_zhx_zhb__ul_jtv_5sl_mcb">
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Default CSV</span> - File that includes comma-separated
                              values. Ignores empty lines in the file.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">RFC4180 CSV</span> - Comma-separated file that strictly
                              follows RFC4180 guidelines.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">MS Excel CSV</span> - Microsoft Excel comma-separated
                              file.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">MySQL CSV</span> - MySQL comma-separated file.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Tab-Separated Values</span> - File that includes
                              tab-separated values.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">PostgreSQL CSV</span> - PostgreSQL comma-separated
                              file.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">PostgreSQL Text</span> - PostgreSQL text file.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Custom</span> - File that uses user-defined delimiter,
                              escape, and quote characters.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Multi Character Delimited</span> - File that uses
                              multiple user-defined characters to delimit fields and lines, and
                              single user-defined escape and quote characters. </li>
                  </ul></div>
                              </dd>
                        
                
                              <dt class="- topic/dt dt dlterm">JSON</dt>
                              <dd class="- topic/dd dd">The destination writes records as JSON data. You can use one of
                                    the following formats:<ul class="- topic/ul ul" id="concept_wld_zhx_zhb__ul_dd1_5y1_wr" data-ofbid="concept_wld_zhx_zhb__ul_dd1_5y1_wr">
                                          <li class="- topic/li li">Array - Each file includes a single array. In the
                                                array, each element is a JSON representation of each
                                                record.</li>
                                          <li class="- topic/li li">Multiple objects - Each file includes multiple JSON
                                                objects. Each object is a JSON representation of a
                                                record. </li>
                                    </ul></dd>
                        
                
                        <dt class="- topic/dt dt dlterm">Protobuf</dt>
                        <dd class="- topic/dd dd">Writes a batch of messages in each file. </dd>
                        <dd class="- topic/dd dd ddexpand">Uses the user-defined message type and the definition of the message
                              type in the descriptor file to generate the messages in the file. </dd>
                        <dd class="- topic/dd dd ddexpand">For information about generating the descriptor file, see <a class="- topic/xref xref" href="../Data_Formats/Protobuf-Prerequisites.html" title="Perform the following prerequisites before reading or writing protobuf data.">Protobuf Data Format Prerequisites</a>.</dd>
                  
                
                        <dt class="- topic/dt dt dlterm">SDC Record</dt>
                        <dd class="- topic/dd dd">The destination writes records in the SDC Record data format. </dd>
                  
                
                        <dt class="- topic/dt dt dlterm">Text</dt>
                        <dd class="- topic/dd dd">The destination writes data from a single text field to the destination
                              system. When you configure the stage, you select the field to use. </dd>
                        <dd class="- topic/dd dd ddexpand">You can configure the characters to use as record separators. By
                              default, the destination uses a UNIX-style line ending (\n) to
                              separate records.</dd>
                        <dd class="- topic/dd dd ddexpand">When a record does not contain the selected text field, the destination
                              can report the missing field as an error or to ignore the missing
                              field. By default, the destination reports an error.</dd>
                        <dd class="- topic/dd dd ddexpand">When configured to ignore a missing text field, the destination can
                              discard the record or write the record separator characters to create
                              an empty line for the record. By default, the destination discards the
                              record.</dd>
                  
                
                        <dt class="- topic/dt dt dlterm">Whole File</dt>
                        <dd class="- topic/dd dd">Streams whole files to the destination system. The destination writes
                              the data to the file and location defined in the stage. If a file of
                              the same name already exists, you can configure the destination to
                              overwrite the existing file or send the current file to error.</dd>
                        <dd class="- topic/dd dd ddexpand">By default, written files use the default access permissions for the
                              destination system. You can specify an expression that defines access
                              permissions. </dd>
                        <dd class="- topic/dd dd ddexpand">You can configure the destination to generate a checksum for the written
                              file and pass checksum information to the destination system in an
                              event record. </dd>
                        <dd class="- topic/dd dd ddexpand">Using this data format requires the following configurations in the
                                    <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Output Files</span> tab: <ul class="- topic/ul ul" id="concept_wld_zhx_zhb__ul_nvb_ldt_kbc" data-ofbid="concept_wld_zhx_zhb__ul_nvb_ldt_kbc">
                                    <li class="- topic/li li">Set the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">File Type</span> property to
                                                <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Whole File</span>.</li>
                                    <li class="- topic/li li">Delete the default value for the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Files
                                                Prefixes</span> property so that the property
                                          is undefined.</li>
                              </ul></dd>
                        <dd class="- topic/dd dd ddexpand">For more information about the whole file data format, see <a class="- topic/xref xref" href="../Data_Formats/WholeFile.html#concept_nfc_qkh_xw">Whole File Data Format</a>.</dd>
                  
            </dl></div>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title13" id="task_nx5_4t5_5hb">
    <h2 class="- topic/title title topictitle2" id="ariaid-title13">Configuring an Azure Data Lake Storage Gen1 Destination</h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc"></p>
        <section class="- topic/section task/context section context">
            <div class="- topic/p p">Configure an Azure Data
                Lake Storage Gen1 destination to write data to Azure Data Lake Storage Gen1. Be sure
                to complete the necessary <a class="- topic/xref xref" href="ADLS-G1-D.html#concept_fn4_qyw_zhb">prerequisites</a> before you configure the destination.<div class="- topic/note note important note_important"><span class="note__title">Important:</span> <span class="- topic/ph ph" id="task_nx5_4t5_5hb__deprecated-ph">This stage is <a class="- topic/xref xref" href="../Pipeline_Design/Deprecated.html#concept_pby_dh2_lpb">deprecated</a> and may be removed in a future release.</span></div></div>
        </section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the Properties panel, on the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">General</span> tab, configure the
                    following properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_yxz_pvs_5x" data-ofbid="task_nx5_4t5_5hb__table_yxz_pvs_5x" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__1">General Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
              <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__1">Name</td>
              <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__2">Stage name.</td>
            </tr>
                                <tr class="- topic/row">
              <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__1">Description</td>
              <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__2">Optional description.</td>
            </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__1"><a class="- topic/xref xref" href="ADLS-G1-D.html#concept_et1_lhx_zhb">Produce Events</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__2">Generates event records when events occur. Use for
                  <a class="- topic/xref xref" href="../Event_Handling/EventFramework-Title.html#concept_cph_5h4_lx">event handling</a>.</td>
                                </tr>
                                <tr class="- topic/row">
              <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__1"><a class="- topic/xref xref" href="../Pipeline_Design/DroppingUnwantedRecords.html#concept_dnj_bkm_vq">Required Fields</a></td>
              <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__2">Fields that must include data for the record to be passed into the stage. <div class="- topic/note note tip note_tip"><span class="note__title">Tip:</span> You might include fields that the stage
                  uses.</div><p class="- topic/p p">Records that do not include all required fields are processed based
                  on the error handling configured for the pipeline.</p></td>
            </tr>
                                <tr class="- topic/row">
              <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__1"><a class="- topic/xref xref" href="../Pipeline_Design/DroppingUnwantedRecords.html#concept_msl_yd4_fs">Preconditions</a></td>
              <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__2">Conditions that must evaluate to TRUE to allow a record to enter the stage for
                processing. Click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Add</span> to create additional preconditions.
                  <p class="- topic/p p">Records that do not meet all preconditions are processed based on the error
                  handling configured for the stage.</p></td>
            </tr>
                                <tr class="- topic/row">
              <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__1"><a class="- topic/xref xref" href="../Pipeline_Design/ErrorHandling.html#concept_atr_j4y_5r">On Record Error</a></td>
              <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_yxz_pvs_5x__entry__2">Error record handling for the stage: <ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_h5l_np2_px" data-ofbid="task_nx5_4t5_5hb__ul_h5l_np2_px">
                  <li class="- topic/li li">Discard - Discards the record.</li>
                  <li class="- topic/li li">Send to Error - Sends the record to the pipeline for error handling.</li>
                  <li class="- topic/li li">Stop Pipeline - Stops the pipeline. </li>
                </ul></td>
            </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">On the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Lake</span> tab, configure the following
                    properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_umv_hsw_zx" data-ofbid="task_nx5_4t5_5hb__table_umv_hsw_zx" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__1">Data Lake Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__1">Application ID</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__2"><span class="- topic/ph ph">Application ID for the Azure Active Directory <span class="- topic/ph ph">Data Collector</span>
                                    application. Also known as the client ID. </span><p class="- topic/p p">For information on accessing the
                                    application ID from the Azure portal, see the <a class="- topic/xref xref" href="https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#get-application-id-and-authentication-key" target="_blank" rel="external noopener">Azure
                                    documentation</a>.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__1">Auth Token Endpoint</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__2"><span class="- topic/ph ph">OAuth 2.0 token endpoint for
                    the Azure Active Directory v1.0 application for <span class="- topic/ph ph">Data Collector</span>. For example:
                        <code class="+ topic/ph pr-d/codeph ph codeph">https://login.microsoftonline.com/&lt;uuid&gt;/oauth2/token.</code></span></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__1">Account FQDN</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__2">The host name of the Azure Data Lake Storage Gen1
                                        account. For example:
                                        <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>&lt;service name&gt;.azuredatalakestore.net</code></pre></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__1">Application Key</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__2"><span class="- topic/ph ph">Authentication key or client secret
                                    for the Azure Active Directory application. Also known as the
                                    client secret.</span><p class="- topic/p p">For information
                                    on accessing the application key from the Azure portal, see the
                                        <a class="- topic/xref xref" href="https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#get-application-id-and-authentication-key" target="_blank" rel="external noopener">Azure
                                    documentation</a>.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__1">Advanced Configuration</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_umv_hsw_zx__entry__2">
                                        <p class="- topic/p p">Additional HDFS properties to pass to the underlying file
                  system. ADLS Gen1 accesses data using the Hadoop FileSystem interface. Specified
                  properties override those in Hadoop configuration files.</p>
                                        <p class="- topic/p p">To add properties, click the <strong class="+ topic/ph hi-d/b ph b">Add </strong>icon and define the
                  HDFS property name and value. Use the property names and values as expected by
                  Hadoop. </p>
                                    </td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">On the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Output Files</span> tab, configure the following
                    properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_tdz_3z4_sy" data-ofbid="task_nx5_4t5_5hb__table_tdz_3z4_sy" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1">Output Files Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1"><a class="- topic/xref xref" href="ADLS-G1-D.html#concept_e5y_2zw_zhb">Directory Template</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Template for creating output
                                        directories. You can use constants, field values, and
                                        datetime variables. <p class="- topic/p p">Output directories are created based
                                            on the smallest datetime variable in the
                                        template.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1">File Type</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Output file type:<ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_pzp_dd1_s5" data-ofbid="task_nx5_4t5_5hb__ul_pzp_dd1_s5">
                                            <li class="- topic/li li">Text files</li>
                                            <li class="- topic/li li">Sequence files</li>
                                            <li class="- topic/li li">Whole files - Select when using the whole file data
                                                format.</li>
                                        </ul></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1">Files Prefix</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Prefix to use for output files. Use when writing to a
                                        directory that receives files from other sources.<p class="- topic/p p">Uses the
                                            prefix <code class="+ topic/ph pr-d/codeph ph codeph">sdc-${sdc:id()}</code> by default. The
                                            prefix evaluates to <code class="+ topic/ph pr-d/codeph ph codeph">sdc-&lt;<span class="- topic/ph ph">Data Collector</span> ID&gt;</code>. </p><p class="- topic/p p">The <span class="- topic/ph ph">Data Collector</span> ID is stored in the following file:
                                                <span class="+ topic/ph sw-d/filepath ph filepath">$SDC_DATA/sdc.id</span>. <span class="- topic/ph ph">For more information about environment variables, see
                              <a class="- topic/xref xref" href="../Configuration/DCEnvironmentConfig.html#concept_rng_qym_qr">Data Collector Environment Configuration</a>.</span></p><div class="- topic/note note note note_note"><span class="note__title">Note:</span> When using the whole file data format,
                                            delete the default prefix value so that this property is
                                            undefined. The File Name Expression property in the Data
                                            Format tab determines the name of the output
                                            file.</div></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1">Files Suffix</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Suffix to use for output files, such as
                                            <code class="+ topic/ph pr-d/codeph ph codeph">txt</code> or <code class="+ topic/ph pr-d/codeph ph codeph">json</code>. When
                                        used, the destination adds a period and the configured
                                        suffix as follows:
                                            <code class="+ topic/ph pr-d/codeph ph codeph">&lt;filename&gt;.&lt;suffix&gt;</code>.<p class="- topic/p p">You can
                                            include periods within the suffix, but do not start the
                                            suffix with a period. Forward slashes are not
                                            allowed.</p><p class="- topic/p p">Not available for the whole file data
                                            format. </p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1"><a class="- topic/xref xref" href="../Pipeline_Design/RecordHeaderAttributes.html#concept_lmn_gdc_1w">Directory in Header</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Indicates that the target directory is defined in record
                                        headers. Use only when the targetDirectory header attribute
                                        is defined for all records. </td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1">Data Time Zone</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Time zone for the destination system. Used to resolve
                                        datetimes in the directory template and evaluate where
                                        records are written.</td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1"><a class="- topic/xref xref" href="ADLS-G1-D.html#concept_wps_x2x_zhb">Time Basis</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Time basis to use for creating
                                        output directories and writing records to the directories.
                                        Use one of the following expressions:<ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_h1q_dd1_s5" data-ofbid="task_nx5_4t5_5hb__ul_h1q_dd1_s5">
                                            <li class="- topic/li li"><code class="+ topic/ph pr-d/codeph ph codeph">${time:now()}</code> - Uses the processing
                                                time as the time basis. </li>
                                            <li class="- topic/li li"><code class="+ topic/ph pr-d/codeph ph codeph">${record:value(&lt;date field
                                                  path&gt;)}</code> - Uses the time associated with
                                                the record as the time basis.</li>
                                        </ul></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1">Max Records in File</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Maximum number of records written to an output file.
                                        Additional records are written to a new file. <p class="- topic/p p">Use 0 to
                                            opt out of this property. </p><p class="- topic/p p">Not available when
                                            using the whole file data format. </p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1">Max File Size (MB)</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Maximum size of an output file. Additional records are
                                        written to a new file. <p class="- topic/p p">Use 0 to opt out of this property.
                                            </p><p class="- topic/p p">Not available when using the whole file data
                                            format. </p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1"><a class="- topic/xref xref" href="ADLS-G1-D.html#concept_kyy_chx_zhb">Idle Timeout</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Maximum time that an output file
                                        can remain idle. After no records are written to a file for
                                        this amount of time, the destination closes the file. Enter
                                        a time in seconds or use the <code class="+ topic/ph pr-d/codeph ph codeph">MINUTES</code> or
                                            <code class="+ topic/ph pr-d/codeph ph codeph">HOURS</code> constant in an expression to
                                        define the time increment.<p class="- topic/p p">Use -1 to set no limit. Default
                                            is 1 hour, defined as follows: <code class="+ topic/ph pr-d/codeph ph codeph">${1 *
                                                HOURS}</code>. </p><p class="- topic/p p">Not available when using the
                                            whole file data format. </p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1">Compression Codec</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Compression type for output files:<ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_o1q_dd1_s5" data-ofbid="task_nx5_4t5_5hb__ul_o1q_dd1_s5">
                                            <li class="- topic/li li">None </li>
                                            <li class="- topic/li li">gzip</li>
                                            <li class="- topic/li li">bzip2</li>
                                            <li class="- topic/li li">Snappy</li>
                                            <li class="- topic/li li">LZ4</li>
                                            <li class="- topic/li li">Other</li>
                                        </ul><div class="- topic/p p">
                                            <div class="- topic/note note note note_note"><span class="note__title">Note:</span> Do not use with Avro data. To compress Avro data,
                                                use the Avro Compression Codec property on the Data
                                                Formats tab.</div>
                                        </div></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1"><a class="- topic/xref xref" href="../Pipeline_Design/RecordHeaderAttributes.html#concept_lmn_gdc_1w">Use Roll Attribute</a>
                                    </td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Checks the record header for the roll header attribute
                                        and closes the current file when the roll attribute exists.
                                            <p class="- topic/p p">Can be used with Max Records in a File and Max File
                                            Size to close files.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1">Roll Attribute Name</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Name of the roll header attribute.<p class="- topic/p p">Default is
                                        roll.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1">Validate Permissions</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__D-entry-ValidatePermissions" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">When you start the
                                        pipeline, the destination tries writing to the configured
                                        directory template to validate permissions. The pipeline
                                        does not start if validation fails.<div class="- topic/note note note note_note"><span class="note__title">Note:</span> Do not use this
                                            option when the directory template uses expressions to
                                            represent the entire directory.</div></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__1"><a class="- topic/xref xref" href="ADLS-G1-D.html#concept_uyx_3bt_13b">Skip File Recovery</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_tdz_3z4_sy__entry__2">Determines whether the
                                        destination performs file recovery after an unexpected stop
                                        of the pipeline. </td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">On the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Format</span> tab, configure the following
                    property:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_hvy_pt3_vx" data-ofbid="task_nx5_4t5_5hb__table_hvy_pt3_vx" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_hvy_pt3_vx__entry__1">Data Format Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_hvy_pt3_vx__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_hvy_pt3_vx__entry__1"><a class="- topic/xref xref" href="ADLS-G2-D.html#concept_lhb_vsk_vhb">Data Format</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_hvy_pt3_vx__entry__2">Format of data to be written. Use one of the following
                                            options:<ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_ffs_xxw_zx" data-ofbid="task_nx5_4t5_5hb__ul_ffs_xxw_zx">
                                            <li class="- topic/li li">Avro</li>
                                            <li class="- topic/li li">Binary</li>
                                            <li class="- topic/li li">Delimited</li>
                                            <li class="- topic/li li">JSON</li>
                                            <li class="- topic/li li">Protobuf</li>
                                            <li class="- topic/li li">SDC Record</li>
                                            <li class="- topic/li li">Text</li>
                                            <li class="- topic/li li">Whole File</li>
                                        </ul></td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">For Avro data, on the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Format</span> tab, configure the
                    following properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_o7j_3nd_9t" data-ofbid="task_nx5_4t5_5hb__table_o7j_3nd_9t" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Avro Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Avro-SchemaLocation">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Avro Schema Location</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">Location of the Avro schema definition to use when
                                        writing data:<ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_mr6_sdg_lx" data-ofbid="task_nx5_4t5_5hb__ul_mr6_sdg_lx">
                                            <li class="- topic/li li">In Pipeline Configuration - Use the schema that you
                                                provide in the stage configuration. </li>
                                            <li class="- topic/li li">In Record Header - Use the schema in the avroSchema
                                                  <a class="- topic/xref xref" href="../Pipeline_Design/RecordHeaderAttributes.html#concept_lmn_gdc_1w">record header attribute</a>. Use only when the
                                                avroSchema attribute is defined for all records. </li>
                                            <li class="- topic/li li">Confluent Schema Registry - Retrieve the schema from
                                                Confluent Schema Registry. </li>
                                        </ul></td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Avro-Schema">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Avro Schema</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">Avro schema definition used to write the data. <p class="- topic/p p">You can
                                            optionally use the <code class="+ topic/ph pr-d/codeph ph codeph">runtime:loadResource</code>
                                            function to load a schema definition stored in a runtime
                                            resource file. </p></td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Avro-RegisterSchema">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Register Schema</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">Registers a new Avro schema with Confluent Schema
                                        Registry.</td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Avro-SchemaRegistryURL">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Schema Registry URLs</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">Confluent Schema Registry URLs used to look up the schema
                                        or to register a new schema. To add a URL, click
                                            <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Add</span> and then enter the URL in the
                                        following
                                        format:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>http://&lt;host name&gt;:&lt;port number&gt;</code></pre></td>
                                </tr>
                                <tr class="- topic/row">
              <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Basic Auth User Info</td>
              <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">User information needed to connect to Confluent Schema Registry when using
                basic authentication. <p class="- topic/p p">Enter the key and secret from the
                    <code class="+ topic/ph pr-d/codeph ph codeph">schema.registry.basic.auth.user.info</code> setting in Schema Registry
                  using the following format: </p><div class="- topic/p p">
                  <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>&lt;key&gt;:&lt;secret&gt;</code></pre>
                </div><div class="- topic/note note tip note_tip"><span class="note__title">Tip:</span> <span class="- topic/ph ph" id="task_nx5_4t5_5hb__ph-runtimeresources_credstores">To
                        secure sensitive information such as user names and passwords, you can use
                              <a class="- topic/xref xref" href="../Pipeline_Configuration/RuntimeValues.html#concept_bs4_5nm_2s" title="Similar to runtime properties, runtime resources are values that you define in a file local to the Data Collector and call from within a pipeline. But with runtime resources, you can restrict the permissions for the files to secure information.">runtime resources</a> or <span class="- topic/ph ph"><a class="- topic/xref xref" href="../Configuration/CredentialStores.html#concept_bt1_bpj_r1b">credential stores.</a></span></span></div></td>
            </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Avro-LookUpSchemaBy">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Look Up Schema By</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">Method used to look up the schema in Confluent Schema
                                            Registry:<ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_t38_m6g_lx" data-ofbid="task_nx5_4t5_5hb__ul_t38_m6g_lx">
                                            <li class="- topic/li li">Subject - Look up the specified Avro schema
                                                subject.</li>
                                            <li class="- topic/li li">Schema ID - Look up the specified Avro schema ID.
                                            </li>
                                        </ul></td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Avro-SchemaSubject">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Schema Subject</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">Avro schema subject to look up or to register in
                                        Confluent Schema Registry.<p class="- topic/p p">If the specified subject to
                                            look up has multiple schema versions, the destination
                                            uses the latest schema version for that subject. To use
                                            an older version, find the corresponding schema ID, and
                                            then set the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Look Up Schema By</span>
                                            property to Schema ID.</p></td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Avro-SchemaID">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Schema ID</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">Avro schema ID to look up in Confluent Schema
                                        Registry.</td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Avro-IncludeSchema">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Include Schema</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">Includes the schema in each file. <div class="- topic/note note note note_note"><span class="note__title">Note:</span> Omitting the
                                            schema definition can improve performance, but requires
                                            the appropriate schema management to avoid losing track
                                            of the schema associated with the data.</div></td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Avro-CompressionCodec">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__1">Avro Compression Codec</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_o7j_3nd_9t__entry__2">The Avro compression type to use. <p class="- topic/p p">When using Avro
                                            compression, do not enable other compression available
                                            in the destination. </p></td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">For binary data, on the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Format</span> tab, configure the
                    following property:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_xct_mbm_gt" data-ofbid="task_nx5_4t5_5hb__table_xct_mbm_gt" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_xct_mbm_gt__entry__1">Binary Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_xct_mbm_gt__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_xct_mbm_gt__entry__1">Binary Field Path</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_xct_mbm_gt__entry__2">Field that contains the binary data.</td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">For delimited data, on the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Format</span> tab, configure the
                    following properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_wb3_2kg_br" data-ofbid="task_nx5_4t5_5hb__table_wb3_2kg_br" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__1">Delimited Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__1">Delimiter Format</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__2">Format for delimited data:<ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_k3j_vvf_jr" data-ofbid="task_nx5_4t5_5hb__ul_k3j_vvf_jr">
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Default CSV</span> - File that includes comma-separated
                              values. Ignores empty lines in the file.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">RFC4180 CSV</span> - Comma-separated file that strictly
                              follows RFC4180 guidelines.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">MS Excel CSV</span> - Microsoft Excel comma-separated
                              file.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">MySQL CSV</span> - MySQL comma-separated file.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Tab-Separated Values</span> - File that includes
                              tab-separated values.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">PostgreSQL CSV</span> - PostgreSQL comma-separated
                              file.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">PostgreSQL Text</span> - PostgreSQL text file.</li>
                        <li class="- topic/li li"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Custom</span> - File that uses user-defined delimiter,
                              escape, and quote characters.</li>
                  </ul></td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-Delim-HeaderLine">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__1">Header Line</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__2">Indicates whether to create a header line.</td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__1">Delimiter Character</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__2">Delimiter character for a custom delimiter format. Select
                                        one of the available options or use Other to enter a custom
                                        character. <p class="- topic/p p">You can enter a Unicode control character
                                            using the format \u<em class="+ topic/ph hi-d/i ph i">NNNN</em>, where â<em class="+ topic/ph hi-d/i ph i">N</em> is a
                                            hexadecimal digit from the numbers 0-9 or the letters
                                            A-F. For example, enter \u0000 to use the null character
                                            as the delimiter or \u2028 to use a line separator as
                                            the delimiter.</p><p class="- topic/p p">Default is the pipe character ( |
                                            ).</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__1">Record Separator String</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__2">Characters to use to separate records. Use any valid Java
                                        string literal. For example, when writing to Windows, you
                                        might use <kbd class="+ topic/ph sw-d/userinput ph userinput">\r\n</kbd> to separate records.
                                            <p class="- topic/p p">Available when using a custom delimiter
                                        format.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__1">Escape Character </td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__2">Escape character for a custom delimiter format. Select
                                        one of the available options or use Other to enter a custom
                                        character. <p class="- topic/p p">Default is the backslash character ( \
                                        ).</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__1">Quote Character</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__2">Quote character for a custom delimiter format. Select one
                                        of the available options or use Other to enter a custom
                                        character. <p class="- topic/p p">Default is the quotation mark character ( "
                                            ).</p></td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-Delim-ReplaceNewLineChar">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__1">Replace New Line Characters</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__2">Replaces new line characters with the configured
                                            string.<p class="- topic/p p">Recommended when writing data as a single line
                                            of text.</p></td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-Delim-NewLineCharReplace">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__1">New Line Character Replacement</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__2">String to replace each new line character. For example,
                                        enter a space to replace each new line character with a
                                        space. <p class="- topic/p p">Leave empty to remove the new line
                                        characters.</p></td>
                                </tr>
                                <tr class="- topic/row">
              <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__1">Charset</td>
              <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_wb3_2kg_br__entry__2">Character set to use when writing data. </td>
            </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">For JSON data, on the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Format</span> tab, configure the
                    following properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_lgq_53c_wr" data-ofbid="task_nx5_4t5_5hb__table_lgq_53c_wr" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_lgq_53c_wr__entry__1">JSON Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_lgq_53c_wr__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_lgq_53c_wr__entry__1">JSON Content</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_lgq_53c_wr__entry__2">Method to write JSON data:<ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_mss_w3c_wr" data-ofbid="task_nx5_4t5_5hb__ul_mss_w3c_wr">
                                            <li class="- topic/li li">JSON Array of Objects - Each file includes a single
                                                array. In the array, each element is a JSON
                                                representation of each record.</li>
                                            <li class="- topic/li li">Multiple JSON Objects - Each file includes multiple
                                                JSON objects. Each object is a JSON representation
                                                of a record.</li>
                                        </ul></td>
                                </tr>
                                <tr class="- topic/row">
              <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_lgq_53c_wr__entry__1">Charset</td>
              <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_lgq_53c_wr__entry__2">Character set to use when writing data. </td>
            </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">For protobuf data, on the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Format</span> tab, configure the
                    following properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_vmt_tdp_45" data-ofbid="task_nx5_4t5_5hb__table_vmt_tdp_45" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_vmt_tdp_45__entry__1">Protobuf Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_vmt_tdp_45__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_vmt_tdp_45__entry__1">Protobuf Descriptor File </td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_vmt_tdp_45__entry__2">Descriptor file (.desc) to use. The descriptor file must
                                        be in the <span class="- topic/ph ph">Data Collector</span> resources directory, <code class="+ topic/ph pr-d/codeph ph codeph">$SDC_RESOURCES</code>.
                                                <p class="- topic/p p"><span class="- topic/ph ph">For more information about environment variables, see
                              <a class="- topic/xref xref" href="../Configuration/DCEnvironmentConfig.html#concept_rng_qym_qr">Data Collector Environment Configuration</a>.</span> For information about generating the descriptor file,
                                            see <a class="- topic/xref xref" href="../Data_Formats/Protobuf-Prerequisites.html" title="Perform the following prerequisites before reading or writing protobuf data.">Protobuf Data Format Prerequisites</a>.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_vmt_tdp_45__entry__1">Message Type</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_vmt_tdp_45__entry__2">Fully-qualified name for the message type to use when
                                        writing data.<p class="- topic/p p">Use the following format:
                                                <code class="+ topic/ph pr-d/codeph ph codeph">&lt;package name&gt;.&lt;message
                                            type&gt;</code>. </p>Use a message type defined in the
                                        descriptor file.</td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">For text data, on the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Format</span> tab, configure the
                    following properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_egv_3df_jr" data-ofbid="task_nx5_4t5_5hb__table_egv_3df_jr" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_egv_3df_jr__entry__1">Text Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_egv_3df_jr__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Text-FieldPath">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_egv_3df_jr__entry__1">Text Field Path</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_egv_3df_jr__entry__2">Field that contains the text data to be written. All data
                                        must be incorporated into the specified field. </td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Text-RecordSeparator">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_egv_3df_jr__entry__1">Record Separator</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_egv_3df_jr__entry__2">Characters to use to separate records. Use any valid Java
                                        string literal. For example, when writing to Windows, you
                                        might use <kbd class="+ topic/ph sw-d/userinput ph userinput">\r\n</kbd> to separate
                                            records.<p class="- topic/p p">By default, the destination uses
                                                <kbd class="+ topic/ph sw-d/userinput ph userinput">\n</kbd>.</p></td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Text-OnMissingField">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_egv_3df_jr__entry__1">On Missing Field</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_egv_3df_jr__entry__2">When a record does not include the text field, determines
                                        whether the destination reports the missing field as an
                                        error or ignores the missing field.</td>
                                </tr>
                                <tr class="- topic/row" id="task_nx5_4t5_5hb__row-D-Text-InsertRecordSeparator">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_egv_3df_jr__entry__1">Insert Record Separator if No Text</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_egv_3df_jr__entry__2">When configured to ignore a missing text field, inserts
                                        the configured record separator string to create an empty
                                            line.<p class="- topic/p p">When not selected, discards records without the
                                            text field.</p></td>
                                </tr>
                                <tr class="- topic/row">
              <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_egv_3df_jr__entry__1">Charset</td>
              <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_egv_3df_jr__entry__2">Character set to use when writing data. </td>
            </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">On the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Late Records</span> tab, configure the following
                    properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_j35_gtj_xhb" data-ofbid="task_nx5_4t5_5hb__table_j35_gtj_xhb" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry align-left colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_j35_gtj_xhb__entry__1"><a class="- topic/xref xref" href="ADLS-G1-D.html#concept_x35_tgx_zhb">Late Records Property</a></th>
                                    <th class="- topic/entry entry align-left colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_j35_gtj_xhb__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry align-left colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_j35_gtj_xhb__entry__1">Late Record Time Limit (secs)</td>
                                    <td class="- topic/entry entry align-left colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_j35_gtj_xhb__entry__2">Time limit for output directories to accept data. <p class="- topic/p p">You
                                            can enter a time in seconds, or use the expression to
                                            enter a time in hours. You can also use MINUTES in the
                                            default expression to define the time in minutes.
                                        </p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry align-left colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_j35_gtj_xhb__entry__1">Late Record Handling</td>
                                    <td class="- topic/entry entry align-left colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_j35_gtj_xhb__entry__2">Determines how to handle late records:<ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_gx4_c12_br" data-ofbid="task_nx5_4t5_5hb__ul_gx4_c12_br">
                                            <li class="- topic/li li">Send to error - Sends the record to the stage for
                                                error handling. </li>
                                            <li class="- topic/li li">Send to late records file - Sends the record to a
                                                late records file.</li>
                                        </ul></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry align-left colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_j35_gtj_xhb__entry__1"><a class="- topic/xref xref" href="ADLS-G1-D.html#concept_e5y_2zw_zhb">Late Record Directory Template</a></td>
                                    <td class="- topic/entry entry align-left colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_j35_gtj_xhb__entry__2">Template for creating late
                                        record directories. You can use constants, field values, and
                                        datetime variables. <p class="- topic/p p">Output directories are created based
                                            on the smallest datetime variable in the
                                        template.</p></td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">For whole files, on the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Format</span> tab, configure the
                    following properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_nx5_4t5_5hb__table_wvv_hcw_1x" data-ofbid="task_nx5_4t5_5hb__table_wvv_hcw_1x" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__1">Whole File Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__1">File Name Expression</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__2">
                                        <p class="- topic/p p">Expression to use for the file names. </p>
                                        <p class="- topic/p p">For tips on how to name files based on input file names,
                                            see <a class="- topic/xref xref" href="../Data_Formats/WholeFile.html#concept_a2s_4jw_1x">Writing Whole Files</a>.</p>
                                    </td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__1">File Exists</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__2">Action to take when a file of the same name already
                                        exists in the output directory. Use one of the following
                                            options:<ul class="- topic/ul ul" id="task_nx5_4t5_5hb__ul_a4s_l5l_zw" data-ofbid="task_nx5_4t5_5hb__ul_a4s_l5l_zw">
                                            <li class="- topic/li li">Send to Error - Handles the record based on stage
                                                error record handling. </li>
                                            <li class="- topic/li li">Overwrite - Overwrites the existing file.</li>
                                        </ul></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__1"><a class="- topic/xref xref" href="../Data_Formats/WholeFile.html#concept_ttm_ywv_1x">Permissions Expression</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__2">Expression that defines the access permissions for output
                                        files. Expressions should evaluate to a symbolic or
                                        numeric/octal representation of the permissions you want to
                                        use. <p class="- topic/p p">By default, with no specified expression, files use
                                            the default permissions of the destination system.
                                            </p><div class="- topic/p p">To use the original source file access
                                            permissions, use the following expression:
                                            <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>${record:value('/fileInfo/permissions')}</code></pre></div></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__1"><a class="- topic/xref xref" href="../Data_Formats/WholeFile.html#concept_ojv_sr4_vx">Include Checksum in Events</a></td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__2">Includes checksum information in whole file event
                                        records. <p class="- topic/p p">Use only when the destination generates event
                                            records. </p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__1">Checksum Algorithm</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_nx5_4t5_5hb__table_wvv_hcw_1x__entry__2">Algorithm to generate the checksum.</td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li></ol></section>
    </div>
</article></article></article></main></div>

                        
                        
                        


                    </div>
                    
                </div>
            </div>


        </div> <nav class="navbar navbar-default wh_footer" data-whc_version="25.0">
  <div class=" footer-container  mx-auto">
    <!-- script for Data Collector, all flavors, but only used when accessed directly, not from portal --><script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-60917135-3', 'auto');
  ga('send', 'pageview');
</script>
  </div>
</nav>

        
        <div id="go2top">
            <span class="oxy-icon oxy-icon-up"></span>
        </div>
        
        <!-- The modal container for images -->
        <div id="modal_img_large" class="modal">
            <span class="close oxy-icon oxy-icon-remove"></span>
            <!-- Modal Content (The Image) -->
            <div id="modal_img_container"></div>
            <!-- Modal Caption (Image Text) -->
            <div id="caption"></div>
        </div>
        
        
        Â© 2023 StreamSets, Inc.

    </body>
</html>